Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/CoreTests/ArenaTests.swift3// Description: Arena events and accumulator on a real captured run, stored comparisons, Cost Intelligence decoding, constellation decoding and the force layout.4import Foundation5import Testing6@testable import SPChatCore78private func arenaEvents(_ sse: String = ArenaFixtures.runSSE + "\n") -> [SSEEvent] {9 var p = SSEParser()10 return p.feed(Data(sse.utf8))11}1213@Suite struct ArenaEventTests {14 @Test func decodesTheCapturedRun() throws {15 let raw = arenaEvents()16 #expect(raw.map(\.event) == ["run", "start", "text", "model_done", "text", "model_done", "done", "end"])17 let events = raw.map(ArenaEvent.init)18 guard case .start(let id, let models) = events[1] else { Issue.record("start expected"); return }19 #expect(id == "cmp_r3qcjmvv6yj5uu9r"); #expect(models.map(\.id) == ["claude-haiku-5-5", "or:google/gemini-4-neon"])20 #expect(models[1].route == "openrouter"); #expect(models[1].creator == "google"); #expect(models[1].logo == "google"); #expect(models[1].displayName == "Gemini 4 Neon")21 #expect(models[0].estimate?.inputTokens == 828); #expect(models[1].estimate?.inputCostUsd == 0.002484)22 #expect(models[0].estimate?.basis == "Estimation avant envoi (≈ 3,5 caractères par jeton, entrée seulement)")23 #expect(events[2] == .text(model: "claude-haiku-5-5", delta: "Bonjour"))24 guard case .modelDone(let neon) = events[5] else { Issue.record("model_done expected"); return }25 #expect(neon.modelId == "or:google/gemini-4-neon"); #expect(neon.servedBy == "OpenRouter · Google"); #expect(neon.exactCost == true); #expect(neon.costUsd == 0.002943)26 #expect(neon.inputTokens == 686); #expect(neon.outputTokens == 59); #expect(neon.reasoningTokens == 54); #expect(neon.latencyMs == 1775); #expect(neon.ttftMs == 1724); #expect(neon.error == nil)27 #expect(events[6] == .done(id: "cmp_r3qcjmvv6yj5uu9r"))28 // `run` and `end` belong to the run envelope, not to the Arena29 #expect(events[0] == .unknown("run")); #expect(events[7] == .unknown("end"))30 }3132 @Test func tolerantOfUnknownAndMalformedEvents() {33 #expect(ArenaEvent(SSEEvent(event: "telemetry", data: #"{"x":1}"#)) == .unknown("telemetry"))34 #expect(ArenaEvent(SSEEvent(event: "text", data: "not json")) == .unknown("text"))35 #expect(ArenaEvent(SSEEvent(event: "text", data: #"{"delta":"a"}"#)) == .unknown("text"))36 #expect(ArenaEvent(SSEEvent(event: "model_done", data: #"{"text":"orphan"}"#)) == .unknown("model_done"))37 #expect(ArenaEvent(SSEEvent(event: "model_error", data: #"{"model":"m"}"#)) == .modelError(model: "m", message: "Erreur"))38 #expect(ArenaEvent(SSEEvent(event: "error", data: #"{"message":"Budget atteint"}"#)) == .failure(message: "Budget atteint"))39 // a start whose models are partly unreadable keeps the readable ones; a missing estimate stays nil40 guard case .start(let id, let models) = ArenaEvent(SSEEvent(event: "start", data: #"{"models":[{"id":"a","estimate":null},{"nope":1},{"id":"b","estimate":{"inputTokens":"12"}}]}"#)) else { Issue.record("start expected"); return }41 #expect(id == nil); #expect(models.map(\.id) == ["a", "b"]); #expect(models[0].estimate == nil); #expect(models[1].estimate?.inputTokens == 12); #expect(models[1].estimate?.inputCostUsd == nil)42 }43}4445@Suite struct ArenaAccumulatorTests {46 @Test func foldsTheCapturedRunIntoColumns() {47 var acc = ArenaAccumulator(modelIds: ["claude-haiku-5-5", "or:google/gemini-4-neon"]) { $0.hasPrefix("or:") ? "openrouter" : "anthropic" }48 let events = arenaEvents().map(ArenaEvent.init)49 // before anything arrives: two waiting columns, in the requested order50 #expect(acc.columns.map(\.modelId) == ["claude-haiku-5-5", "or:google/gemini-4-neon"]); #expect(acc.columns.allSatisfy { !$0.done && $0.text.isEmpty })5152 #expect(acc.apply(events[0]) == false) // run53 acc.apply(events[1]) // start: identity + « estimé » figures54 #expect(acc.comparisonId == "cmp_r3qcjmvv6yj5uu9r"); #expect(!acc.finished)55 #expect(acc.columns[0].estimate?.inputCostUsd == 0.00008280000000000001); #expect(acc.columns[1].route == "openrouter"); #expect(acc.columns[1].displayName == "Gemini 4 Neon")5657 acc.apply(events[2])58 #expect(acc.columns[0].text == "Bonjour"); #expect(!acc.columns[0].done); #expect(acc.columns[1].text.isEmpty)59 acc.apply(events[3])60 #expect(acc.columns[0].done); #expect(acc.columns[0].hasAnswer); #expect(!acc.columns[0].isBilledCost); #expect(acc.columns[0].costUsd == 0.000108)61 #expect(acc.columns[0].inputTokens == 1045); #expect(acc.columns[0].outputTokens == 7); #expect(acc.columns[0].ttftMs == 639); #expect(acc.columns[0].latencyMs == 641); #expect(acc.columns[0].servedBy == nil)62 #expect(acc.comparableAnswers == 1)6364 for e in events[4...] { acc.apply(e) }65 #expect(acc.finished); #expect(acc.failure == nil); #expect(acc.comparableAnswers == 2)66 let neon = acc.columns[1]67 #expect(neon.text == "bonjour"); #expect(neon.isBilledCost); #expect(neon.costUsd == 0.002943); #expect(neon.servedBy == "OpenRouter · Google"); #expect(neon.reasoningTokens == 54)68 #expect(neon.estimate?.inputTokens == 828, "the pre-run estimate is kept next to the measured figures")69 }7071 @Test func samEventsSameColumns() {72 let events = arenaEvents().map(ArenaEvent.init)73 var a = ArenaAccumulator(modelIds: ["claude-haiku-5-5", "or:google/gemini-4-neon"]), b = a74 for e in events { a.apply(e) }; for e in events { b.apply(e) }75 #expect(a == b)76 }7778 @Test func errorsLateDeltasAndUnannouncedModels() {79 var acc = ArenaAccumulator(modelIds: ["a", "b", "a"])80 #expect(acc.columns.map(\.modelId) == ["a", "b"], "a duplicated id gives one column")81 acc.apply(.reasoning(model: "a", delta: "je pense")); acc.apply(.text(model: "a", delta: "Bon")); acc.apply(.text(model: "a", delta: "jour"))82 #expect(acc.columns[0].reasoning == "je pense"); #expect(acc.columns[0].text == "Bonjour")83 #expect(acc.apply(.text(model: "a", delta: "")) == false)8485 // a failing model: error shown, the partial text survives a model_done without text86 acc.apply(.text(model: "b", delta: "début")); acc.apply(.modelError(model: "b", message: "Flux interrompu avant la fin"))87 let failed = ArenaResult(JSONValue.parse(Data(#"{"modelId":"b","text":"","usage":{"input":10,"output":2},"costUsd":0.0001,"exactCost":false,"latencyMs":900,"ttftMs":null,"error":"Flux interrompu avant la fin"}"#.utf8)) ?? .null)88 acc.apply(.modelDone(failed!))89 #expect(acc.columns[1].error == "Flux interrompu avant la fin"); #expect(acc.columns[1].text == "début"); #expect(acc.columns[1].done); #expect(!acc.columns[1].hasAnswer); #expect(acc.columns[1].ttftMs == nil)9091 // a replayed delta after model_done (re-attached stream) never duplicates text92 acc.apply(.modelDone(ArenaResult(JSONValue.parse(Data(#"{"modelId":"a","text":"Bonjour","usage":{"input":"5","output":"1"},"latencyMs":300}"#.utf8))!)!))93 #expect(acc.apply(.text(model: "a", delta: "jour")) == false); #expect(acc.columns[0].text == "Bonjour"); #expect(acc.columns[0].inputTokens == 5); #expect(acc.columns[0].exactCost == nil)9495 // an event for a model that was never announced adds a column instead of being lost96 acc.apply(.text(model: "c", delta: "surprise"))97 #expect(acc.columns.map(\.modelId) == ["a", "b", "c"])98 acc.apply(.failure(message: "Budget atteint")); #expect(acc.failure == "Budget atteint"); #expect(!acc.finished)99 acc.close(); #expect(acc.columns.allSatisfy { $0.done }); #expect(acc.columns.filter { $0.hasAnswer }.map(\.modelId) == ["a", "c"], "b failed; the text of a stopped column is kept")100 }101102 @Test func storedComparisonAndHistory() throws {103 let j = try #require(JSONValue.parse(Data(ArenaFixtures.storedComparison.utf8)))104 let c = try #require(ArenaComparison(j["comparison"]))105 #expect(c.id == "cmp_r3qcjmvv6yj5uu9r"); #expect(c.prompt == "Réponds par un seul mot : bonjour"); #expect(c.modelIds == ["claude-haiku-5-5", "or:google/gemini-4-neon"])106 #expect(c.columns.count == 2); #expect(c.columns.allSatisfy { $0.hasAnswer }); #expect(c.columns[1].isBilledCost); #expect(c.columns[0].exactCost == false)107 #expect(c.judge == nil); #expect(c.preferredModel == nil); #expect(c.conversationId == nil); #expect(c.createdAt != nil)108 #expect(ArenaComparison(JSONValue.null) == nil)109110 let judge = try #require(ArenaJudge(JSONValue.parse(Data(#"{"judgeModel":"claude-haiku-5-5","text":"RANKING: B > A","mapping":{"B":"m2","A":"m1"},"costUsd":0.0004}"#.utf8))))111 #expect(judge.mapping.map(\.label) == ["A", "B"]); #expect(judge.label(for: "m2") == "B"); #expect(judge.disclaimer.contains("opinion"))112 let item = try #require(ArenaHistoryItem(JSONValue.parse(Data(#"{"id":"cmp_1","prompt":"p","model_ids":["a","b"],"preferred_model":"b","created_at":"2026-11-19T14:39:44.398Z","cost":"0.003051"}"#.utf8))!))113 #expect(item.cost == 0.003051); #expect(item.preferredModel == "b"); #expect(item.modelIds.count == 2)114 }115116 @Test func endpointsSendTheContract() async throws {117 let t = MockTransport(); let api = makeClient(t)118 t.on("POST", "/api/compare", .sse([ArenaFixtures.runSSE + "\n"]))119 t.on("PATCH", "/api/compare", .json(200, #"{"ok":true}"#))120 t.on("POST", "/api/conversations/conv_1/adopt", .json(201, #"{"ok":true,"messageId":"msg_9"}"#))121 t.on("GET", "/api/compare/export", .data(200, Data("# Arène".utf8), headers: ["Content-Type": "text/markdown; charset=utf-8", "content-disposition": #"attachment; filename="arene-2026-11-19.md""#]))122 t.on("GET", "/api/compare", .json(200, ArenaFixtures.storedComparison))123124 guard case .events(_, let bytes) = try await api.openArena(prompt: "Salut", modelIds: ["a", "or:b/c"], effort: "low") else { Issue.record("SSE expected"); return }125 var parser = SSEParser(); var acc = ArenaAccumulator(modelIds: ["claude-haiku-5-5", "or:google/gemini-4-neon"])126 for try await chunk in bytes { for e in parser.feed(chunk) { acc.apply(ArenaEvent(e)) } }127 #expect(acc.finished); #expect(acc.comparableAnswers == 2)128 let sentBody = try #require(t.requests[0].body); let sent = try #require(JSONValue.parse(sentBody))129 #expect(sent["prompt"]?.stringValue == "Salut"); #expect(sent["modelIds"]?.arrayValue?.compactMap(\.stringValue) == ["a", "or:b/c"]); #expect(sent["settings"]?["effort"]?.stringValue == "low")130131 try await api.preferArenaAnswer(comparisonId: "cmp_1", modelId: "or:b/c")132 #expect(t.requests[1].body.flatMap(JSONValue.parse) == ["id": "cmp_1", "preferredModel": "or:b/c"])133 let adopted = try await api.adoptArenaAnswer(conversationId: "conv_1", comparisonId: "cmp_1", modelId: "a"); #expect(adopted == "msg_9")134 #expect(t.requests[2].body.flatMap(JSONValue.parse) == ["comparisonId": "cmp_1", "modelId": "a"])135 let file = try await api.exportArena(comparisonId: "cmp_1", format: .markdown)136 #expect(file.filename == "arene-2026-11-19.md"); #expect(file.mime == "text/markdown"); #expect(t.requests[3].query?.contains("format=md") == true)137 let stored = try await api.arenaComparison("cmp_r3qcjmvv6yj5uu9r"); #expect(stored?.columns.count == 2)138 }139}140141@Suite struct CostIntelligenceTests {142 @Test func decodesTheRealReport() throws {143 let u = try JSONDecoder.api().decode(UsageReport.self, from: Data(ArenaFixtures.usage.utf8))144 #expect(u.totals.requests == 965); #expect(u.byRoute.count == 4)145 let or = try #require(u.byRoute.first { $0.route == "openrouter" })146 #expect(or.provider == "openrouter"); #expect(or.costExact == 0.064968); #expect(or.requests == 7)147 #expect(u.byRoute.filter { $0.route == "direct" }.map(\.provider) == ["anthropic", "openai", "xai"])148 #expect(u.byConversation.count == 6); #expect(u.byConversation[0].title == "Histoire des phares à travers les âges"); #expect(u.byConversation[0].lastAt != nil)149 #expect(u.costBasis?.exact == 0.064968); #expect(u.costBasis?.estimated == 0.634193); #expect(u.costBasis?.note?.hasPrefix("« exact »") == true)150 #expect(u.byProject.first?.project == "—"); #expect(u.byProject.first?.projectId == nil); #expect(u.monthly.count == 1); #expect(u.monthly[0].cost == 0.699161)151 #expect(u.range?.from != nil)152 }153154 @Test func olderServerAndBrokenRowsNeverFail() throws {155 // 0.3 server: none of the new keys156 let old = #"{"totals":{"requests":1,"cost":0.1},"daily":[],"byModel":[],"byKind":[],"spending":{"today":0,"month":0,"limits":{"dailyLimitUsd":10,"monthlyLimitUsd":150,"alertAtPercent":80}}}"#157 let u = try JSONDecoder.api().decode(UsageReport.self, from: Data(old.utf8))158 #expect(u.byRoute.isEmpty); #expect(u.byConversation.isEmpty); #expect(u.costBasis == nil); #expect(u.monthly.isEmpty); #expect(u.range == nil)159 // new keys with unreadable rows and wrong types: the readable rows survive160 let odd = #"{"totals":{"requests":1,"cost":0.1},"daily":[],"byModel":[],"byKind":[],"spending":{"today":0,"month":0,"limits":{"dailyLimitUsd":10,"monthlyLimitUsd":150,"alertAtPercent":80}},"byRoute":[{"provider":"openrouter","cost":"0.5","cost_exact":"0.5","requests":"3"},42],"byConversation":[{"title":"sans id"},{"conversation_id":"c1","cost":1}],"costBasis":"n/a","monthly":{"x":1},"byProject":null}"#161 let v = try JSONDecoder.api().decode(UsageReport.self, from: Data(odd.utf8))162 #expect(v.byRoute.count == 1); #expect(v.byRoute[0].route == "openrouter"); #expect(v.byRoute[0].cost == 0.5); #expect(v.byRoute[0].requests == 3)163 #expect(v.byConversation.map(\.conversationId) == ["c1"]); #expect(v.costBasis == nil); #expect(v.monthly.isEmpty); #expect(v.byProject.isEmpty)164 }165}166167@Suite struct ConstellationTests {168 @Test func decodesTheRealGraph() throws {169 let g = try JSONDecoder.api().decode(Constellation.self, from: Data(ArenaFixtures.constellation.utf8))170 #expect(g.nodes.count == 6); #expect(g.links.count == 7); #expect(g.links.allSatisfy { $0.kind == "topic" })171 #expect(g.basis["topic"] == "Proximité sémantique : similarité cosinus entre les plongements moyens des messages (seuil 0,55, 3 voisins au plus).")172 #expect(g.basis["branch"] == "Lien de bifurcation enregistré."); #expect(g.basis["project"] == "Conversations du même projet, par ordre chronologique.")173 let n = try #require(g.nodes.first { $0.id == "conv_757ydbh4j2f7lqk6" })174 #expect(n.title == "Essai mobile 375"); #expect(n.favorite); #expect(n.messages == 2); #expect(n.costUsd == 0.000131); #expect(n.projectColor == nil); #expect(n.lastAt != nil)175 #expect(g.neighbours(of: "conv_757ydbh4j2f7lqk6") == ["conv_ch5pacgypxiwsuma"])176 }177178 @Test func dropsDanglingLinksAndUnreadableRows() throws {179 let json = ##"{"nodes":[{"id":"a","title":"A","messages":"3","costUsd":"0.5","projectId":"p","projectColor":"#5A45E0"},{"title":"sans id"},{"id":"b"},{"id":"a","title":"doublon"}],"links":[{"source":"a","target":"b","kind":"branch","weight":1},{"source":"a","target":"zzz","kind":"topic","weight":0.9},{"source":"a","target":"a"},{"source":"b","target":"a","kind":"wormhole"}],"projects":[{"id":"p","name":"Projet","color":"#5A45E0","count":"1"}],"basis":{"topic":"t","extra":7}}"##180 let g = try JSONDecoder.api().decode(Constellation.self, from: Data(json.utf8))181 #expect(g.nodes.map(\.id) == ["a", "b"]); #expect(g.nodes[0].messages == 3); #expect(g.nodes[0].costUsd == 0.5); #expect(g.nodes[1].title.isEmpty)182 #expect(g.links.map(\.kind) == ["branch", "wormhole"]); #expect(g.links[1].weight == 1)183 #expect(g.projects.first?.count == 1); #expect(g.basis == ["topic": "t"])184 let empty = try JSONDecoder.api().decode(Constellation.self, from: Data("{}".utf8))185 #expect(empty.nodes.isEmpty); #expect(empty.links.isEmpty); #expect(empty.basis.isEmpty)186 }187188 @Test func endpointClampsTheLimit() async throws {189 let t = MockTransport(); let api = makeClient(t)190 t.on("GET", "/api/constellation", .json(200, ArenaFixtures.constellation))191 _ = try await api.constellation(limit: 5000, projectId: "proj_1"); _ = try await api.constellation(limit: 1)192 #expect(t.requests[0].query?.contains("limit=300") == true); #expect(t.requests[0].query?.contains("project=proj_1") == true)193 #expect(t.requests[1].query?.contains("limit=10") == true); #expect(t.requests[1].query?.contains("project") == false)194 }195}196197@Suite struct ForceLayoutTests {198 /// A ring of clusters: `clusters` groups of `size` fully linked nodes, consecutive groups joined by one weak edge.199 private func graph(clusters: Int, size: Int) -> (radii: [Double], edges: [ForceLayout.Edge]) {200 var edges: [ForceLayout.Edge] = []201 for c in 0..<clusters {202 let base = c * size203 for i in 0..<size { for j in (i + 1)..<size { edges.append(.init(base + i, base + j, strength: 0.9)) } }204 edges.append(.init(base, ((c + 1) % clusters) * size, strength: 0.2))205 }206 return ((0..<(clusters * size)).map { 6 + Double($0 % 5) * 2 }, edges)207 }208 private func distance(_ a: ForceLayout.Point, _ b: ForceLayout.Point) -> Double { ((a.x - b.x) * (a.x - b.x) + (a.y - b.y) * (a.y - b.y)).squareRoot() }209210 @Test func deterministicForASeed() {211 let g = graph(clusters: 5, size: 6)212 var c = ForceLayout.Configuration(); c.seed = 42213 let a = ForceLayout.run(radii: g.radii, edges: g.edges, width: 900, height: 600, configuration: c)214 let b = ForceLayout.run(radii: g.radii, edges: g.edges, width: 900, height: 600, configuration: c)215 #expect(a == b)216 c.seed = 43217 #expect(ForceLayout.run(radii: g.radii, edges: g.edges, width: 900, height: 600, configuration: c).positions != a.positions)218 }219220 @Test func convergesFiniteAndInBounds() {221 for (clusters, size, w, h) in [(5, 6, 900.0, 600.0), (1, 2, 300.0, 200.0), (20, 8, 1400.0, 900.0), (12, 1, 500.0, 500.0)] {222 let g = graph(clusters: clusters, size: size)223 let r = ForceLayout.run(radii: g.radii, edges: g.edges, width: w, height: h)224 #expect(r.positions.count == g.radii.count)225 #expect(r.residual < 1, "settled: last move \(r.residual) pt for \(clusters)×\(size)")226 for (i, p) in r.positions.enumerated() {227 #expect(p.x.isFinite && p.y.isFinite)228 #expect(p.x >= g.radii[i] && p.x <= w - g.radii[i] && p.y >= g.radii[i] && p.y <= h - g.radii[i], "node \(i) inside the bounds")229 }230 }231 }232233 @Test func linkedNodesEndCloserThanUnlinkedOnes() {234 let g = graph(clusters: 4, size: 6)235 let r = ForceLayout.run(radii: g.radii, edges: g.edges, width: 1000, height: 700)236 var inside = 0.0, insideCount = 0.0, across = 0.0, acrossCount = 0.0237 for i in 0..<24 { for j in (i + 1)..<24 {238 let d = distance(r.positions[i], r.positions[j])239 if i / 6 == j / 6 { inside += d; insideCount += 1 } else { across += d; acrossCount += 1 }240 } }241 #expect(inside / insideCount < 0.6 * (across / acrossCount), "clusters are visible")242 // nodes never sit on top of each other243 for i in 0..<24 { for j in (i + 1)..<24 { #expect(distance(r.positions[i], r.positions[j]) > 4) } }244 }245246 @Test func degenerateInputs() {247 #expect(ForceLayout.run(radii: [], edges: [], width: 100, height: 100).positions.isEmpty)248 #expect(ForceLayout.run(radii: [10], edges: [.init(0, 0)], width: 200, height: 100).positions == [ForceLayout.Point(x: 100, y: 50)])249 // invalid edges, NaN inputs, zero size: still finite250 let odd = ForceLayout.run(radii: [5, .nan, -3, 8], edges: [.init(0, 9), .init(-1, 2), .init(1, 1), .init(0, 1, strength: .nan), .init(2, 3, strength: 7)], width: 0, height: .infinity)251 #expect(odd.positions.count == 4); #expect(odd.positions.allSatisfy { $0.x.isFinite && $0.y.isFinite })252 var c = ForceLayout.Configuration(); c.iterations = 0253 let one = ForceLayout.run(radii: [4, 4, 4], edges: [], width: 300, height: 300, configuration: c)254 #expect(one.iterations == 1); #expect(one.positions.allSatisfy { $0.x.isFinite && $0.y.isFinite })255 }256257 @Test func threeHundredNodesStaysFast() {258 let g = graph(clusters: 30, size: 10)259 // fewer iterations than the default: this test shares the machine with the other suites' timing tests260 var c = ForceLayout.Configuration(); c.iterations = 100261 let clock = ContinuousClock(); var result: ForceLayout.Result?262 let elapsed = clock.measure { result = ForceLayout.run(radii: g.radii, edges: g.edges, width: 2000, height: 1400, configuration: c) }263 #expect(result?.positions.count == 300)264 #expect(elapsed < .seconds(8), "300 nodes laid out in \(elapsed) (debug build)")265 }266}267