Repository Public
1//2// SP Chat iOS3// File: SPChat-iOS/Tests/ArenaTests.swift4// Description: Arena — decoding of the `/api/compare` SSE vocabulary, the ArenaState reducer, ArenaSession against5// a scripted server, and one end-to-end comparison against the development backend.6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9//1011import Foundation12import Testing13@testable import SPChat1415private enum ArenaFixtures {16 static let start = #"{"id":"cmp_test1","models":[{"id":"or:google/gemini-4-neon","provider":"openrouter","route":"openrouter","creator":"google","logo":"google","displayName":"Gemini 4 Neon","estimate":{"inputTokens":825,"inputCostUsd":0.002475,"basis":"Estimation avant envoi"}},{"id":"claude-haiku-5-5","provider":"anthropic","route":"direct","creator":"anthropic","logo":"anthropic","displayName":"Claude Haiku 5.5","estimate":null}]}"#17 static let haikuDone = #"{"modelId":"claude-haiku-5-5","provider":"anthropic","route":"direct","creator":"anthropic","logo":"anthropic","displayName":"Claude Haiku 5.5","servedBy":null,"text":"Pong","usage":{"input":1041,"output":5,"cached":0,"reasoning":0},"costUsd":0.0001066,"exactCost":false,"latencyMs":614,"ttftMs":611,"error":null}"#18 static let neonDone = #"{"modelId":"or:google/gemini-4-neon","provider":"openrouter","route":"openrouter","creator":"google","logo":"google","displayName":"Gemini 4 Neon","servedBy":"OpenRouter · Google","text":"Pong !","reasoning":"court","usage":{"input":684,"output":38,"cached":0,"reasoning":33,"costUsd":0.002622},"costUsd":0.002622,"exactCost":true,"latencyMs":2832,"ttftMs":2746,"error":null}"#19 static func ev(_ name: String, _ json: String) -> ArenaEvent { ArenaEvent(sse: SSEEvent(event: name, data: json, id: nil)) }2021 /// A complete numbered run, as the backend emits it.22 static func run(end: Bool = true) -> [String] {23 var e: [String] = ["retry: 2000\n\n"]; var i = 024 func add(_ n: String, _ j: String) { e.append(SSE.event(n, j, id: i)); i += 1 }25 add("run", #"{"runId":"run_cmp1"}"#); add("start", start)26 add("text", #"{"model":"claude-haiku-5-5","delta":"Po"}"#); add("text", #"{"model":"claude-haiku-5-5","delta":"ng"}"#)27 add("reasoning", #"{"model":"or:google/gemini-4-neon","delta":"court"}"#)28 add("text", #"{"model":"or:google/gemini-4-neon","delta":"Pong !"}"#)29 if end { add("model_done", haikuDone); add("model_done", neonDone); add("done", #"{"id":"cmp_test1"}"#); add("end", "{}") }30 return e31 }32 static var models: [AIModel] {33 [AIModel(id: "claude-haiku-5-5", provider: "anthropic", displayName: "Claude Haiku 5.5"), AIModel(id: "or:google/gemini-4-neon", provider: "openrouter", displayName: "Gemini 4 Neon", creator: "google", status: "catalog")]34 }35}3637@Suite struct ArenaEventTests {38 @Test func decodesEveryEventOfTheStream() {39 #expect(ArenaFixtures.ev("run", #"{"runId":"run_x"}"#) == .run(id: "run_x"))40 guard case .start(let id, let cols) = ArenaFixtures.ev("start", ArenaFixtures.start) else { Issue.record("start"); return }41 #expect(id == "cmp_test1" && cols.map(\.id) == ["or:google/gemini-4-neon", "claude-haiku-5-5"])42 #expect(cols.first?.route == "openrouter" && cols.first?.logo == "google" && cols.first?.estimatedInputCostUsd == 0.002475 && cols.first?.estimateBasis == "Estimation avant envoi")43 #expect(cols.last?.estimatedInputCostUsd == nil && cols.last?.isDone == false)44 #expect(ArenaFixtures.ev("text", #"{"model":"m","delta":"a"}"#) == .text(model: "m", delta: "a"))45 #expect(ArenaFixtures.ev("reasoning", #"{"model":"m","delta":"r"}"#) == .reasoning(model: "m", delta: "r"))46 #expect(ArenaFixtures.ev("model_error", #"{"model":"m","message":"clé refusée"}"#) == .modelError(model: "m", message: "clé refusée"))47 guard case .modelDone(let c) = ArenaFixtures.ev("model_done", ArenaFixtures.neonDone) else { Issue.record("model_done"); return }48 #expect(c.id == "or:google/gemini-4-neon" && c.isDone && c.text == "Pong !" && c.reasoning == "court" && c.servedBy == "OpenRouter · Google")49 #expect(c.inputTokens == 684 && c.outputTokens == 38 && c.costUsd == 0.002622 && c.exactCost && c.latencyMs == 2832 && c.ttftMs == 2746 && c.error == nil)50 #expect(ArenaFixtures.ev("done", #"{"id":"cmp_1"}"#) == .done(id: "cmp_1"))51 #expect(ArenaFixtures.ev("end", "{}") == .end)52 #expect(ArenaFixtures.ev("error", #"{"message":"budget"}"#) == .error("budget"))53 }5455 @Test func unknownOrMalformedEventsNeverBreakTheStream() {56 #expect(ArenaFixtures.ev("something_new", #"{"x":1}"#) == .other)57 #expect(ArenaFixtures.ev("model_done", "not json") == .other)58 #expect(ArenaFixtures.ev("text", "{}") == .text(model: "", delta: ""))59 guard case .start(_, let cols) = ArenaFixtures.ev("start", #"{"id":"c","models":[{"provider":"x"},{"id":"ok"}]}"#) else { Issue.record("start"); return }60 #expect(cols.map(\.id) == ["ok"] && cols.first?.displayName == "ok")61 }6263 @Test func judgementDecodesItsMappingAndDisclaimer() {64 let j = try? JSONDecoder().decode(JSONValue.self, from: Data(#"{"judgeModel":"claude-haiku-5-5","text":"RANKING: A > B","mapping":{"A":"m1","B":"m2"},"costUsd":0.001,"disclaimer":"Une opinion."}"#.utf8))65 let judgement = j.flatMap(ArenaJudgement.init)66 #expect(judgement?.mapping == ["A": "m1", "B": "m2"] && judgement?.disclaimer == "Une opinion." && judgement?.judgeModel == "claude-haiku-5-5")67 #expect(ArenaJudgement(.object(["judgeModel": .string("x")])) == nil)68 }69}7071@Suite struct ArenaStateTests {72 private func played(_ events: [(String, String)]) -> ArenaState {73 var s = ArenaState(prompt: "ping", models: ArenaFixtures.models)74 for (n, j) in events { s.apply(ArenaFixtures.ev(n, j)) }75 return s76 }7778 @Test func startTakesTheServerOrderAndKeepsTextAlreadyReceived() {79 var s = ArenaState(prompt: "ping", models: ArenaFixtures.models)80 #expect(s.columns.map(\.id) == ["claude-haiku-5-5", "or:google/gemini-4-neon"] && s.isRunning && s.id == nil)81 s.apply(.text(model: "claude-haiku-5-5", delta: "tôt"))82 s.apply(ArenaFixtures.ev("run", #"{"runId":"run_1"}"#))83 s.apply(ArenaFixtures.ev("start", ArenaFixtures.start))84 #expect(s.id == "cmp_test1" && s.runId == "run_1" && s.columns.map(\.id) == ["or:google/gemini-4-neon", "claude-haiku-5-5"])85 #expect(s.columns.last?.text == "tôt" && s.columns.first?.estimatedInputCostUsd == 0.002475)86 }8788 @Test func deltasGoToTheirOwnColumn() {89 let s = played([("start", ArenaFixtures.start), ("text", #"{"model":"claude-haiku-5-5","delta":"Po"}"#), ("text", #"{"model":"or:google/gemini-4-neon","delta":"X"}"#),90 ("text", #"{"model":"claude-haiku-5-5","delta":"ng"}"#), ("reasoning", #"{"model":"or:google/gemini-4-neon","delta":"r"}"#), ("text", #"{"model":"inconnu","delta":"?"}"#)])91 #expect(s.columns.first { $0.id == "claude-haiku-5-5" }?.text == "Pong")92 let neon = s.columns.first { $0.id == "or:google/gemini-4-neon" }93 #expect(neon?.text == "X" && neon?.reasoning == "r" && s.finishedCount == 0 && s.totalCostUsd == nil)94 }9596 @Test func modelDoneKeepsTheEstimateAndTellsBilledFromEstimatedCosts() {97 var s = played([("start", ArenaFixtures.start), ("text", #"{"model":"or:google/gemini-4-neon","delta":"Pong"}"#), ("model_done", ArenaFixtures.neonDone)])98 let neon = s.columns.first { $0.id == "or:google/gemini-4-neon" }99 #expect(neon?.isDone == true && neon?.exactCost == true && neon?.costUsd == 0.002622 && neon?.estimatedInputCostUsd == 0.002475 && neon?.estimateBasis != nil && neon?.text == "Pong !")100 #expect(s.finishedCount == 1 && s.allCostsExact && s.isRunning)101 s.apply(ArenaFixtures.ev("model_done", ArenaFixtures.haikuDone))102 #expect(s.finishedCount == 2 && !s.allCostsExact) // one billed, one estimated: the total is an estimate103 #expect(abs((s.totalCostUsd ?? 0) - 0.0027286) < 1e-9 && s.fastestFirstToken == "claude-haiku-5-5")104 s.apply(ArenaFixtures.ev("done", #"{"id":"cmp_test1"}"#))105 #expect(!s.isRunning && s.runId == nil && s.id == "cmp_test1" && s.isStored)106 }107108 @Test func aModelErrorStaysInItsColumnAndTheOthersGoOn() {109 var s = played([("start", ArenaFixtures.start), ("model_error", #"{"model":"or:google/gemini-4-neon","message":"Clé refusée"}"#), ("text", #"{"model":"claude-haiku-5-5","delta":"ok"}"#)])110 #expect(s.columns.first?.error == "Clé refusée" && s.columns.last?.error == nil && s.columns.last?.text == "ok" && s.error == nil && s.isRunning)111 let failed = #"{"modelId":"or:google/gemini-4-neon","displayName":"Gemini 4 Neon","text":"","usage":{"input":0,"output":0},"costUsd":0,"exactCost":false,"latencyMs":90,"ttftMs":null,"error":"Clé refusée"}"#112 s.apply(ArenaFixtures.ev("model_done", failed))113 #expect(s.columns.first?.isDone == true && s.columns.first?.error == "Clé refusée" && s.columns.first?.ttftMs == nil && s.fastestFirstToken == nil)114 }115116 @Test func aGlobalErrorOrAnInterruptedStreamEndsTheRunWithoutInventingResults() {117 var s = played([("start", ArenaFixtures.start), ("error", #"{"message":"Limite atteinte"}"#)])118 #expect(!s.isRunning && s.error == "Limite atteinte")119 var t = played([("run", #"{"runId":"run_9"}"#), ("start", ArenaFixtures.start), ("text", #"{"model":"claude-haiku-5-5","delta":"dé"}"#), ("model_done", ArenaFixtures.neonDone)])120 t.interrupt(reason: "Arrêtée")121 #expect(!t.isRunning && t.runId == nil && !t.isStored && t.columns.allSatisfy(\.isDone))122 let haiku = t.columns.first { $0.id == "claude-haiku-5-5" }123 #expect(haiku?.error == "Arrêtée" && haiku?.text == "dé" && haiku?.costUsd == nil)124 #expect(t.columns.first { $0.id == "or:google/gemini-4-neon" }?.error == nil)125 s.apply(.end)126 #expect(!s.isRunning)127 }128}129130@Suite(.serialized) @MainActor struct ArenaSessionTests {131 private func session(host: String) -> ArenaSession {132 let env = TestEnv.make(host: host)133 let known = Dictionary(uniqueKeysWithValues: ArenaFixtures.models.map { ($0.id, $0) })134 let s = ArenaSession(api: env.api, model: { known[$0] })135 s.setModels(ArenaFixtures.models.map(\.id))136 return s137 }138 nonisolated private static func json(_ body: Data?) -> [String: JSONValue] { (body.flatMap { try? JSONDecoder().decode([String: JSONValue].self, from: $0) }) ?? [:] }139140 @Test func theLineUpAcceptsTwoToSixKnownModels() {141 let s = session(host: "arena-lineup.test")142 s.setModels(["claude-haiku-5-5", "inconnu", "claude-haiku-5-5"])143 #expect(s.modelIds == ["claude-haiku-5-5"])144 s.prompt = "ping"145 #expect(!s.canStart) // one model is not a comparison146 s.add("or:google/gemini-4-neon")147 #expect(s.canStart && s.models.count == 2)148 s.prompt = " "149 #expect(!s.canStart)150 s.remove("claude-haiku-5-5")151 #expect(s.modelIds == ["or:google/gemini-4-neon"])152 }153154 @Test func aComparisonStreamsThenIsStoredAndCanBePreferredJudgedAdoptedAndExported() async throws {155 let host = "arena-run.test"156 StubProtocol.install(host: host) { req, body in157 let path = req.url?.path ?? ""158 switch (req.httpMethod ?? "GET", path) {159 case ("POST", "/api/compare"): return .sse(ArenaFixtures.run(), delay: 0.02)160 case ("PATCH", "/api/compare"): return .json(#"{"ok":true}"#)161 case ("POST", "/api/compare/judge"): return .json(#"{"judge":{"judgeModel":"claude-haiku-5-5","text":"RANKING: A > B","mapping":{"A":"or:google/gemini-4-neon","B":"claude-haiku-5-5"},"costUsd":0.0004,"disclaimer":"Évaluation générée par un modèle."}}"#)162 case ("GET", "/api/compare/export"): return StubProtocol.Response(headers: ["Content-Type": "text/markdown; charset=utf-8"], chunks: [(0, Data("# Arène multi-modèles\n\nPong".utf8))])163 case ("POST", "/api/conversations/conv_test1/adopt"): return .json(#"{"ok":true}"#)164 default: return .json("{}")165 }166 }167 let s = session(host: host)168 s.prompt = " Réponds : ping "; s.effort = .low; s.conversationId = "conv_test1"169 s.start()170 #expect(s.isRunning && s.prompt.isEmpty && s.state?.prompt == "Réponds : ping" && !s.isStored)171 #expect(await TestEnv.wait { !s.isRunning })172 let state = try #require(s.state)173 #expect(state.id == "cmp_test1" && !state.isRunning && state.columns.map(\.id) == ["or:google/gemini-4-neon", "claude-haiku-5-5"])174 #expect(state.columns.first?.text == "Pong !" && state.columns.first?.reasoning == "court" && state.columns.first?.exactCost == true && state.columns.last?.text == "Pong" && state.columns.last?.exactCost == false)175 #expect(s.isStored && s.error == nil)176177 // the request: trimmed prompt, the line-up in order, effort, the conversation it came from178 let post = try #require(StubProtocol.requests(host: host).first { $0.url?.path == "/api/compare" && $0.httpMethod == "POST" })179 #expect(post.value(forHTTPHeaderField: "Accept") == "text/event-stream")180181 await s.prefer("claude-haiku-5-5")182 #expect(s.state?.preferred == "claude-haiku-5-5")183 await s.judge(with: "claude-haiku-5-5")184 #expect(s.state?.judgement?.mapping["A"] == "or:google/gemini-4-neon" && s.state?.judgement?.disclaimer?.isEmpty == false && !s.isJudging)185 #expect(await s.adopt("or:google/gemini-4-neon") && s.adoptedModel == "or:google/gemini-4-neon")186 let url = try #require(await s.export(format: "md"))187 let exported = try String(contentsOf: url, encoding: .utf8)188 #expect(url.pathExtension == "md" && exported.contains("Arène multi-modèles"))189 try? FileManager.default.removeItem(at: url)190 let paths = StubProtocol.requests(host: host).map { "\($0.httpMethod ?? "") \($0.url?.path ?? "")" }191 #expect(paths.contains("PATCH /api/compare") && paths.contains("POST /api/compare/judge") && paths.contains("POST /api/conversations/conv_test1/adopt") && paths.contains("GET /api/compare/export"))192193 // same prompt again, then a clean slate194 s.rerun()195 #expect(s.isRunning && s.state?.prompt == "Réponds : ping" && s.adoptedModel == nil)196 #expect(await TestEnv.wait { !s.isRunning })197 s.reset()198 #expect(s.state == nil && s.modelIds.count == 2)199 }200201 @Test func theRequestBodyCarriesPromptModelsEffortAndConversation() async throws {202 let host = "arena-body.test"203 let seen = NSLockBox<[String: JSONValue]>([:])204 StubProtocol.install(host: host) { req, body in205 if req.url?.path == "/api/compare" { let j = Self.json(body); seen.withLock { $0 = j } }206 return .sse(ArenaFixtures.run(), delay: 0)207 }208 let s = session(host: host)209 s.prompt = "ping"; s.effort = .high; s.conversationId = "conv_test1"; s.projectId = "proj_1"210 s.start()211 #expect(await TestEnv.wait { !s.isRunning })212 let body = seen.withLock { $0 }213 #expect(body["prompt"]?.stringValue == "ping" && body["modelIds"]?.arrayValue?.compactMap(\.stringValue) == ["claude-haiku-5-5", "or:google/gemini-4-neon"])214 #expect(body["settings"]?["effort"]?.stringValue == "high" && body["conversationId"]?.stringValue == "conv_test1" && body["projectId"]?.stringValue == "proj_1")215 }216217 @Test func stopCancelsTheServerRunAndClosesTheOpenColumnsAsTheyStand() async throws {218 let host = "arena-stop.test"219 StubProtocol.install(host: host) { req, _ in220 if req.url?.path == "/api/compare" { return .sse(ArenaFixtures.run(end: false) + Array(repeating: ": ping\n\n", count: 60), delay: 0.05) }221 return .json(#"{"ok":true,"cancelled":true}"#)222 }223 let s = session(host: host)224 s.prompt = "ping"225 s.start()226 #expect(await TestEnv.wait { s.state?.columns.contains { $0.text == "Pong" } == true })227 s.stop()228 #expect(!s.isRunning && s.state?.isRunning == false && !s.isStored) // never reported `done`: nothing to prefer or export229 #expect(s.state?.columns.allSatisfy { $0.isDone && $0.error != nil && $0.costUsd == nil } == true)230 #expect(await TestEnv.wait { StubProtocol.requests(host: host).contains { $0.url?.path == "/api/chat/cancel" } })231 await s.prefer("claude-haiku-5-5")232 #expect(s.state?.preferred == nil && !StubProtocol.requests(host: host).contains { $0.httpMethod == "PATCH" })233 }234235 @Test func aRefusedOrDroppedComparisonIsReportedNotInvented() async throws {236 let refused = "arena-refused.test"237 StubProtocol.install(host: refused) { _, _ in .json(#"{"error":"Limite quotidienne atteinte","code":"budget"}"#, status: 402) }238 let a = session(host: refused)239 a.prompt = "ping"; a.start()240 #expect(await TestEnv.wait { !a.isRunning })241 #expect(a.error != nil && a.state?.id == nil && !a.isStored && a.state?.columns.allSatisfy { $0.text.isEmpty } == true)242243 let dropped = "arena-dropped.test"244 StubProtocol.install(host: dropped) { _, _ in .sse(ArenaFixtures.run(end: false), delay: 0.01, drop: true) }245 let b = session(host: dropped)246 b.prompt = "ping"; b.start()247 #expect(await TestEnv.wait { !b.isRunning })248 #expect(b.state?.isRunning == false && !b.isStored && b.state?.columns.allSatisfy { $0.isDone && $0.error != nil } == true)249 #expect(b.state?.columns.first { $0.id == "claude-haiku-5-5" }?.text == "Pong") // what was received is kept250 }251}252253@Suite(.serialized, .enabled(if: Live.enabled)) @MainActor struct ArenaLiveTests {254 @Test func aRealComparisonOfTwoRoutesIsMeasuredPreferredAndExported() async throws {255 let env = Live.environment()256 await env.catalog.refreshModels()257 let s = ArenaSession(api: env.api, model: { [catalog = env.catalog] in catalog.model($0) })258 s.setModels(["or:google/gemini-4-neon", "claude-haiku-5-5"])259 #expect(s.models.count == 2 && s.models.first?.isOpenRouter == true)260 s.prompt = "Réponds en un mot : ping"; s.effort = .low261 s.start()262 #expect(await TestEnv.wait(timeout: 120) { !s.isRunning })263 let state = try #require(s.state)264 #expect(s.isStored && state.id?.hasPrefix("cmp_") == true && s.error == nil)265 for c in state.columns {266 #expect(c.isDone && c.error == nil && !c.text.isEmpty && (c.outputTokens ?? 0) > 0 && (c.latencyMs ?? 0) > 0 && c.ttftMs != nil && c.costUsd != nil)267 }268 let neon = try #require(state.columns.first { $0.id == "or:google/gemini-4-neon" })269 let haiku = try #require(state.columns.first { $0.id == "claude-haiku-5-5" })270 #expect(neon.route == "openrouter" && neon.logo == "google" && neon.exactCost && neon.servedBy?.contains("OpenRouter") == true && neon.estimatedInputCostUsd != nil)271 #expect(haiku.route == "direct" && !haiku.exactCost) // Anthropic direct: tokens × list price272 await s.prefer("claude-haiku-5-5")273 #expect(s.state?.preferred == "claude-haiku-5-5" && s.error == nil)274 let url = try #require(await s.export(format: "md"))275 let md = try String(contentsOf: url, encoding: .utf8)276 #expect(md.contains("Arène multi-modèles") && md.contains("claude-haiku-5-5 ★") && md.contains("(facturé)") && md.contains("(estimé)"))277 let jsonURL = try #require(await s.export(format: "json"))278 let exported = try JSONDecoder().decode(JSONValue.self, from: Data(contentsOf: jsonURL))279 #expect(exported.str("preferred_model") == "claude-haiku-5-5" && exported.list("results").count == 2)280 try? FileManager.default.removeItem(at: url); try? FileManager.default.removeItem(at: jsonURL)281 }282}283