Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/AppTests/ArenaScreenTests.swift3// Description: Hosted tests of the Arena: a real captured run streamed into columns, prefer / adopt / export / judge calls, pre-selection, and off-screen snapshots (light + dark).4import SwiftUI5import XCTest6@testable import SPChat7@testable import SPChatCore89@MainActor final class ArenaScreenTests: XCTestCase {10 /// Environment whose catalogue is the real (trimmed) direct + OpenRouter catalogue.11 private func makeEnv(_ t: MockTransport) async -> AppEnvironment {12 t.on("GET", "/api/models", .json(200, ArenaAppFixtures.models))13 let env = Fixture.env(t)14 await env.catalog.load()15 return env16 }17 private func storedComparison() throws -> ArenaComparison {18 try XCTUnwrap(ArenaComparison(JSONValue.parse(Data(ArenaAppFixtures.comparison.utf8))?["comparison"]))19 }20 private var runFrames: [String] { (ArenaAppFixtures.runSSE + "\n").components(separatedBy: "\n\n").filter { !$0.isEmpty }.map { $0 + "\n\n" } }2122 func testRunStreamsIntoColumnsThenPreferAdoptExport() async throws {23 let t = MockTransport(); let env = await makeEnv(t)24 t.on("POST", "/api/compare", .sse(runFrames))25 t.on("GET", "/api/compare", .json(200, ArenaAppFixtures.history))26 t.on("PATCH", "/api/compare", .json(200, #"{"ok":true}"#))27 t.on("POST", "/api/conversations", .json(201, "{\"conversation\":\(Fixture.conversationRow("conv_new", "Moyenne et médiane"))}"))28 t.on("POST", "/api/conversations/conv_new/adopt", .json(201, #"{"ok":true,"messageId":"msg_adopted"}"#))29 t.on("POST", "/api/conversations/conv_old/adopt", .json(409, #"{"error":"Une réponse est en cours dans cette conversation.","code":"busy"}"#))30 t.on("GET", "/api/conversations", .json(200, "{\"conversations\":[\(Fixture.conversationRow("conv_new", "Moyenne et médiane"))]}"))31 t.on("GET", "/api/compare/export", .data(200, Data("# Arène multi-modèles — 2026-11-19".utf8), headers: ["Content-Type": "text/markdown; charset=utf-8", "content-disposition": #"attachment; filename="arene-2026-11-19.md""#]))3233 let model = ArenaViewModel()34 model.applyDefaults(models: env.catalog.selectableModels)35 XCTAssertEqual(model.selected, ["claude-haiku-5-5", "or:google/gemini-4-neon"], "favourites are pre-selected, any route")36 XCTAssertEqual(model.judgeModel, "claude-haiku-5-5"); XCTAssertFalse(model.canRun, "no prompt yet")37 model.selected = ["claude-haiku-5-5", "or:google/gemini-4-neon", "gpt-5.4-mini"]38 model.prompt = "Explique en quatre points très courts la différence entre moyenne et médiane, puis donne un petit tableau d’exemple (5 valeurs)."39 XCTAssertTrue(model.canRun)40 model.start(env: env)41 XCTAssertTrue(model.running); XCTAssertEqual(model.columns.map(\.modelId), model.selected); XCTAssertFalse(model.canAct)42 await Fixture.wait("run to end") { !model.running }4344 XCTAssertEqual(model.comparisonId, "cmp_wx0zogqoecmww5ek"); XCTAssertTrue(model.canAct); XCTAssertNil(model.errorText); XCTAssertFalse(model.connectionLost)45 XCTAssertEqual(model.columns.count, 3); XCTAssertTrue(model.columns.allSatisfy { $0.done && $0.hasAnswer }); XCTAssertEqual(model.comparableAnswers, 3)46 let neon = try XCTUnwrap(model.columns.first { $0.modelId == "or:google/gemini-4-neon" })47 XCTAssertTrue(neon.isBilledCost); XCTAssertEqual(neon.costUsd, 0.024354); XCTAssertEqual(neon.servedBy, "OpenRouter · Google"); XCTAssertEqual(neon.reasoningTokens, 1303)48 let haiku = try XCTUnwrap(model.columns.first { $0.modelId == "claude-haiku-5-5" })49 XCTAssertFalse(haiku.isBilledCost); XCTAssertEqual(haiku.inputTokens, 1078); XCTAssertEqual(haiku.ttftMs, 5526); XCTAssertNotNil(haiku.estimate?.basis)50 XCTAssertTrue(haiku.text.contains("médiane"))51 let sent = try XCTUnwrap(t.requests.first { $0.method == "POST" && $0.path == "/api/compare" }?.body.flatMap(JSONValue.parse))52 XCTAssertEqual(sent["modelIds"]?.arrayValue?.compactMap(\.stringValue), ["claude-haiku-5-5", "or:google/gemini-4-neon", "gpt-5.4-mini"]); XCTAssertNil(sent["settings"])53 await Fixture.wait("history") { model.history.count == 2 }5455 // prefer56 model.prefer("gpt-5.4-mini", env: env)57 XCTAssertEqual(model.preferred, "gpt-5.4-mini")58 await Fixture.wait("PATCH") { t.requests.contains { $0.method == "PATCH" && $0.path == "/api/compare" } }59 XCTAssertEqual(t.requests.first { $0.method == "PATCH" }?.body.flatMap(JSONValue.parse), ["id": "cmp_wx0zogqoecmww5ek", "preferredModel": "gpt-5.4-mini"])6061 // adopt into a new conversation: created with the model, then the stored answer is brought in (no model call)62 let adopted = await model.adopt("claude-haiku-5-5", into: nil, env: env)63 XCTAssertEqual(adopted, "conv_new"); XCTAssertEqual(model.preferred, "claude-haiku-5-5"); XCTAssertNil(model.adopting)64 XCTAssertEqual(t.requests.first { $0.path == "/api/conversations" && $0.method == "POST" }?.body.flatMap(JSONValue.parse)?["modelId"]?.stringValue, "claude-haiku-5-5")65 XCTAssertEqual(t.requests.first { $0.path == "/api/conversations/conv_new/adopt" }?.body.flatMap(JSONValue.parse), ["comparisonId": "cmp_wx0zogqoecmww5ek", "modelId": "claude-haiku-5-5"])66 // a refusal by the server is reported, nothing is opened67 let refused = await model.adopt("gpt-5.4-mini", into: "conv_old", env: env)68 XCTAssertNil(refused); XCTAssertEqual(env.toasts.current?.isError, true); XCTAssertEqual(model.preferred, "claude-haiku-5-5")6970 // export: the server's document71 let file = try await model.exportFile(.markdown, api: env.api)72 XCTAssertEqual(file.filename, "arene-2026-11-19.md"); XCTAssertTrue(String(decoding: file.data, as: UTF8.self).hasPrefix("# Arène")); XCTAssertFalse(model.exporting)73 XCTAssertTrue(t.requests.contains { $0.path == "/api/compare/export" && $0.query?.contains("id=cmp_wx0zogqoecmww5ek") == true && $0.query?.contains("format=md") == true })7475 model.clearResults(); XCTAssertTrue(model.columns.isEmpty); XCTAssertFalse(model.canAct)76 do { _ = try await model.exportFile(.json, api: env.api); XCTFail("nothing to export") } catch {}77 }7879 func testBudgetRefusalAndJudge() async throws {80 let t = MockTransport(); let env = await makeEnv(t)81 t.on("POST", "/api/compare", .json(402, #"{"error":"Limite quotidienne atteinte (10 $).","code":"budget"}"#))82 t.on("GET", "/api/compare", .json(200, #"{"comparisons":[]}"#))83 t.on("POST", "/api/compare/judge", .json(200, #"{"judge":{"judgeModel":"claude-haiku-5-5","text":"RANKING: A > B > C","mapping":{"A":"claude-haiku-5-5","B":"or:google/gemini-4-neon","C":"gpt-5.4-mini"},"costUsd":0.0021,"disclaimer":"Évaluation générée par un modèle : une opinion, pas une vérité objective."}}"#))84 let model = ArenaViewModel(); model.selected = ["claude-haiku-5-5", "gpt-5.4-mini"]; model.prompt = "Salut"85 model.start(env: env)86 await Fixture.wait("refusal") { !model.running }87 XCTAssertEqual(model.errorText, "Budget atteint — Limite quotidienne atteinte (10 $)."); XCTAssertTrue(model.columns.isEmpty); XCTAssertNil(model.comparisonId)8889 model.show(try storedComparison(), known: Set(env.catalog.selectableModels.map(\.id)))90 XCTAssertTrue(model.fromHistory); XCTAssertEqual(model.selected.count, 3); XCTAssertEqual(model.preferred, "claude-haiku-5-5"); XCTAssertNotNil(model.judge); XCTAssertNil(model.errorText)91 model.judgeModel = "claude-haiku-5-5"; model.runJudge(env: env)92 XCTAssertTrue(model.judging)93 await Fixture.wait("judge") { !model.judging }94 XCTAssertEqual(model.judge?.text, "RANKING: A > B > C"); XCTAssertEqual(model.judge?.label(for: "gpt-5.4-mini"), "C")95 XCTAssertEqual(t.requests.first { $0.path == "/api/compare/judge" }?.body.flatMap(JSONValue.parse), ["id": "cmp_wx0zogqoecmww5ek", "judgeModel": "claude-haiku-5-5"])96 }9798 func testPreselectionAndSelectionRules() async {99 let t = MockTransport(); let env = await makeEnv(t)100 let model = ArenaViewModel()101 model.preselect(["or:qwen/qwen4-max", "ghost-model", "claude-opus-5-5", "or:qwen/qwen4-max"], catalog: env.catalog)102 XCTAssertEqual(model.selected, ["or:qwen/qwen4-max", "claude-opus-5-5"], "any route; unknown and duplicated ids skipped")103 model.applyDefaults(models: env.catalog.selectableModels)104 XCTAssertEqual(model.selected, ["or:qwen/qwen4-max", "claude-opus-5-5"], "the hand-over is not replaced by the favourites")105 model.preselect([], catalog: env.catalog); model.preselect(["ghost"], catalog: env.catalog)106 XCTAssertEqual(model.selected.count, 2)107 for id in env.catalog.selectableModels.map(\.id) { if !model.selected.contains(id) { model.toggle(id) } }108 XCTAssertEqual(model.selected.count, ArenaViewModel.maxModels, "six models at most")109 model.toggle("claude-opus-5-5"); XCTAssertFalse(model.selected.contains("claude-opus-5-5"))110 XCTAssertEqual(ArenaColumns.visibleCount(width: 1600, columns: 6), 4); XCTAssertEqual(ArenaColumns.visibleCount(width: 1200, columns: 6), 3)111 XCTAssertEqual(ArenaColumns.visibleCount(width: 800, columns: 6), 2); XCTAssertEqual(ArenaColumns.visibleCount(width: 1600, columns: 2), 2)112 }113114 func testSnapshots() async throws {115 let t = MockTransport(); let env = await makeEnv(t)116 t.on("GET", "/api/compare", .json(200, ArenaAppFixtures.history))117 t.on("GET", "/api/conversations", .json(200, "{\"conversations\":[\(Fixture.conversationRow("conv_1", "Histoire des phares en mer"))]}"))118 await env.conversations.refresh()119120 // a finished three-column comparison, judged, one answer preferred121 let finished = ArenaViewModel(); finished.show(try storedComparison(), known: Set(env.catalog.selectableModels.map(\.id))); finished.judgeModel = "claude-haiku-5-5"122 for dark in [false, true] {123 let r = Snapshot.render(CompareScreen(window: WindowModel(), model: finished).environment(env), name: "arena-finished-\(dark ? "dark" : "light")", size: CGSize(width: 1440, height: 940), dark: dark)124 XCTAssertGreaterThan(r.opaqueFraction, 0.95)125 }126127 // mid-stream: the real run cut after its first deltas, the connection left open128 let cut = runFrames.prefix(60)129 t.on("POST", "/api/compare", .sse(Array(cut), hang: true)); t.on("POST", "/api/chat/cancel", .json(200, #"{"ok":true}"#))130 let live = ArenaViewModel(); live.selected = ["claude-haiku-5-5", "or:google/gemini-4-neon", "gpt-5.4-mini"]; live.prompt = finished.prompt131 live.start(env: env)132 await Fixture.wait("first deltas") { live.columns.contains { !$0.text.isEmpty } && live.comparisonId != nil }133 XCTAssertTrue(live.running); XCTAssertTrue(live.columns.contains { !$0.done })134 XCTAssertNotNil(live.columns[0].estimate?.inputCostUsd, "« estimé » figures are available before the answers")135 for dark in [false, true] {136 Snapshot.render(CompareScreen(window: WindowModel(), model: live).environment(env), name: "arena-streaming-\(dark ? "dark" : "light")", size: CGSize(width: 1280, height: 820), dark: dark)137 }138 live.detach()139 await Fixture.wait("detached") { !live.running }140 XCTAssertTrue(live.columns.allSatisfy(\.done))141142 // empty state and the model picker over the whole catalogue143 Snapshot.render(CompareScreen(window: WindowModel(), model: ArenaViewModel()).environment(env), name: "arena-empty-light", size: CGSize(width: 1100, height: 700))144 for dark in [false, true] {145 let p = Snapshot.render(ArenaModelPicker(model: finished).environment(env).background(DS.C.surface), name: "arena-picker-\(dark ? "dark" : "light")", dark: dark)146 XCTAssertEqual(p.size.width, 460, accuracy: 1)147 }148 print("SNAPSHOTS: \(Snapshot.directory.path)")149 }150}151