SPB Git

Repository Public

ArenaTests.swift 20.8 KB 267 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/CoreTests/ArenaTests.swift3// Description: Arena events and accumulator on a real captured run, stored comparisons, Cost Intelligence decoding, constellation decoding and the force layout.4import Foundation5import Testing6@testable import SPChatCore78private func arenaEvents(_ sse: String = ArenaFixtures.runSSE + "\n") -> [SSEEvent] {9    var p = SSEParser()10    return p.feed(Data(sse.utf8))11}1213@Suite struct ArenaEventTests {14    @Test func decodesTheCapturedRun() throws {15        let raw = arenaEvents()16        #expect(raw.map(\.event) == ["run", "start", "text", "model_done", "text", "model_done", "done", "end"])17        let events = raw.map(ArenaEvent.init)18        guard case .start(let id, let models) = events[1] else { Issue.record("start expected"); return }19        #expect(id == "cmp_r3qcjmvv6yj5uu9r"); #expect(models.map(\.id) == ["claude-haiku-5-5", "or:google/gemini-4-neon"])20        #expect(models[1].route == "openrouter"); #expect(models[1].creator == "google"); #expect(models[1].logo == "google"); #expect(models[1].displayName == "Gemini 4 Neon")21        #expect(models[0].estimate?.inputTokens == 828); #expect(models[1].estimate?.inputCostUsd == 0.002484)22        #expect(models[0].estimate?.basis == "Estimation avant envoi (≈ 3,5 caractères par jeton, entrée seulement)")23        #expect(events[2] == .text(model: "claude-haiku-5-5", delta: "Bonjour"))24        guard case .modelDone(let neon) = events[5] else { Issue.record("model_done expected"); return }25        #expect(neon.modelId == "or:google/gemini-4-neon"); #expect(neon.servedBy == "OpenRouter · Google"); #expect(neon.exactCost == true); #expect(neon.costUsd == 0.002943)26        #expect(neon.inputTokens == 686); #expect(neon.outputTokens == 59); #expect(neon.reasoningTokens == 54); #expect(neon.latencyMs == 1775); #expect(neon.ttftMs == 1724); #expect(neon.error == nil)27        #expect(events[6] == .done(id: "cmp_r3qcjmvv6yj5uu9r"))28        // `run` and `end` belong to the run envelope, not to the Arena29        #expect(events[0] == .unknown("run")); #expect(events[7] == .unknown("end"))30    }3132    @Test func tolerantOfUnknownAndMalformedEvents() {33        #expect(ArenaEvent(SSEEvent(event: "telemetry", data: #"{"x":1}"#)) == .unknown("telemetry"))34        #expect(ArenaEvent(SSEEvent(event: "text", data: "not json")) == .unknown("text"))35        #expect(ArenaEvent(SSEEvent(event: "text", data: #"{"delta":"a"}"#)) == .unknown("text"))36        #expect(ArenaEvent(SSEEvent(event: "model_done", data: #"{"text":"orphan"}"#)) == .unknown("model_done"))37        #expect(ArenaEvent(SSEEvent(event: "model_error", data: #"{"model":"m"}"#)) == .modelError(model: "m", message: "Erreur"))38        #expect(ArenaEvent(SSEEvent(event: "error", data: #"{"message":"Budget atteint"}"#)) == .failure(message: "Budget atteint"))39        // a start whose models are partly unreadable keeps the readable ones; a missing estimate stays nil40        guard case .start(let id, let models) = ArenaEvent(SSEEvent(event: "start", data: #"{"models":[{"id":"a","estimate":null},{"nope":1},{"id":"b","estimate":{"inputTokens":"12"}}]}"#)) else { Issue.record("start expected"); return }41        #expect(id == nil); #expect(models.map(\.id) == ["a", "b"]); #expect(models[0].estimate == nil); #expect(models[1].estimate?.inputTokens == 12); #expect(models[1].estimate?.inputCostUsd == nil)42    }43}4445@Suite struct ArenaAccumulatorTests {46    @Test func foldsTheCapturedRunIntoColumns() {47        var acc = ArenaAccumulator(modelIds: ["claude-haiku-5-5", "or:google/gemini-4-neon"]) { $0.hasPrefix("or:") ? "openrouter" : "anthropic" }48        let events = arenaEvents().map(ArenaEvent.init)49        // before anything arrives: two waiting columns, in the requested order50        #expect(acc.columns.map(\.modelId) == ["claude-haiku-5-5", "or:google/gemini-4-neon"]); #expect(acc.columns.allSatisfy { !$0.done && $0.text.isEmpty })5152        #expect(acc.apply(events[0]) == false) // run53        acc.apply(events[1])                   // start: identity + « estimé » figures54        #expect(acc.comparisonId == "cmp_r3qcjmvv6yj5uu9r"); #expect(!acc.finished)55        #expect(acc.columns[0].estimate?.inputCostUsd == 0.00008280000000000001); #expect(acc.columns[1].route == "openrouter"); #expect(acc.columns[1].displayName == "Gemini 4 Neon")5657        acc.apply(events[2])58        #expect(acc.columns[0].text == "Bonjour"); #expect(!acc.columns[0].done); #expect(acc.columns[1].text.isEmpty)59        acc.apply(events[3])60        #expect(acc.columns[0].done); #expect(acc.columns[0].hasAnswer); #expect(!acc.columns[0].isBilledCost); #expect(acc.columns[0].costUsd == 0.000108)61        #expect(acc.columns[0].inputTokens == 1045); #expect(acc.columns[0].outputTokens == 7); #expect(acc.columns[0].ttftMs == 639); #expect(acc.columns[0].latencyMs == 641); #expect(acc.columns[0].servedBy == nil)62        #expect(acc.comparableAnswers == 1)6364        for e in events[4...] { acc.apply(e) }65        #expect(acc.finished); #expect(acc.failure == nil); #expect(acc.comparableAnswers == 2)66        let neon = acc.columns[1]67        #expect(neon.text == "bonjour"); #expect(neon.isBilledCost); #expect(neon.costUsd == 0.002943); #expect(neon.servedBy == "OpenRouter · Google"); #expect(neon.reasoningTokens == 54)68        #expect(neon.estimate?.inputTokens == 828, "the pre-run estimate is kept next to the measured figures")69    }7071    @Test func samEventsSameColumns() {72        let events = arenaEvents().map(ArenaEvent.init)73        var a = ArenaAccumulator(modelIds: ["claude-haiku-5-5", "or:google/gemini-4-neon"]), b = a74        for e in events { a.apply(e) }; for e in events { b.apply(e) }75        #expect(a == b)76    }7778    @Test func errorsLateDeltasAndUnannouncedModels() {79        var acc = ArenaAccumulator(modelIds: ["a", "b", "a"])80        #expect(acc.columns.map(\.modelId) == ["a", "b"], "a duplicated id gives one column")81        acc.apply(.reasoning(model: "a", delta: "je pense")); acc.apply(.text(model: "a", delta: "Bon")); acc.apply(.text(model: "a", delta: "jour"))82        #expect(acc.columns[0].reasoning == "je pense"); #expect(acc.columns[0].text == "Bonjour")83        #expect(acc.apply(.text(model: "a", delta: "")) == false)8485        // a failing model: error shown, the partial text survives a model_done without text86        acc.apply(.text(model: "b", delta: "début")); acc.apply(.modelError(model: "b", message: "Flux interrompu avant la fin"))87        let failed = ArenaResult(JSONValue.parse(Data(#"{"modelId":"b","text":"","usage":{"input":10,"output":2},"costUsd":0.0001,"exactCost":false,"latencyMs":900,"ttftMs":null,"error":"Flux interrompu avant la fin"}"#.utf8)) ?? .null)88        acc.apply(.modelDone(failed!))89        #expect(acc.columns[1].error == "Flux interrompu avant la fin"); #expect(acc.columns[1].text == "début"); #expect(acc.columns[1].done); #expect(!acc.columns[1].hasAnswer); #expect(acc.columns[1].ttftMs == nil)9091        // a replayed delta after model_done (re-attached stream) never duplicates text92        acc.apply(.modelDone(ArenaResult(JSONValue.parse(Data(#"{"modelId":"a","text":"Bonjour","usage":{"input":"5","output":"1"},"latencyMs":300}"#.utf8))!)!))93        #expect(acc.apply(.text(model: "a", delta: "jour")) == false); #expect(acc.columns[0].text == "Bonjour"); #expect(acc.columns[0].inputTokens == 5); #expect(acc.columns[0].exactCost == nil)9495        // an event for a model that was never announced adds a column instead of being lost96        acc.apply(.text(model: "c", delta: "surprise"))97        #expect(acc.columns.map(\.modelId) == ["a", "b", "c"])98        acc.apply(.failure(message: "Budget atteint")); #expect(acc.failure == "Budget atteint"); #expect(!acc.finished)99        acc.close(); #expect(acc.columns.allSatisfy { $0.done }); #expect(acc.columns.filter { $0.hasAnswer }.map(\.modelId) == ["a", "c"], "b failed; the text of a stopped column is kept")100    }101102    @Test func storedComparisonAndHistory() throws {103        let j = try #require(JSONValue.parse(Data(ArenaFixtures.storedComparison.utf8)))104        let c = try #require(ArenaComparison(j["comparison"]))105        #expect(c.id == "cmp_r3qcjmvv6yj5uu9r"); #expect(c.prompt == "Réponds par un seul mot : bonjour"); #expect(c.modelIds == ["claude-haiku-5-5", "or:google/gemini-4-neon"])106        #expect(c.columns.count == 2); #expect(c.columns.allSatisfy { $0.hasAnswer }); #expect(c.columns[1].isBilledCost); #expect(c.columns[0].exactCost == false)107        #expect(c.judge == nil); #expect(c.preferredModel == nil); #expect(c.conversationId == nil); #expect(c.createdAt != nil)108        #expect(ArenaComparison(JSONValue.null) == nil)109110        let judge = try #require(ArenaJudge(JSONValue.parse(Data(#"{"judgeModel":"claude-haiku-5-5","text":"RANKING: B > A","mapping":{"B":"m2","A":"m1"},"costUsd":0.0004}"#.utf8))))111        #expect(judge.mapping.map(\.label) == ["A", "B"]); #expect(judge.label(for: "m2") == "B"); #expect(judge.disclaimer.contains("opinion"))112        let item = try #require(ArenaHistoryItem(JSONValue.parse(Data(#"{"id":"cmp_1","prompt":"p","model_ids":["a","b"],"preferred_model":"b","created_at":"2026-11-19T14:39:44.398Z","cost":"0.003051"}"#.utf8))!))113        #expect(item.cost == 0.003051); #expect(item.preferredModel == "b"); #expect(item.modelIds.count == 2)114    }115116    @Test func endpointsSendTheContract() async throws {117        let t = MockTransport(); let api = makeClient(t)118        t.on("POST", "/api/compare", .sse([ArenaFixtures.runSSE + "\n"]))119        t.on("PATCH", "/api/compare", .json(200, #"{"ok":true}"#))120        t.on("POST", "/api/conversations/conv_1/adopt", .json(201, #"{"ok":true,"messageId":"msg_9"}"#))121        t.on("GET", "/api/compare/export", .data(200, Data("# Arène".utf8), headers: ["Content-Type": "text/markdown; charset=utf-8", "content-disposition": #"attachment; filename="arene-2026-11-19.md""#]))122        t.on("GET", "/api/compare", .json(200, ArenaFixtures.storedComparison))123124        guard case .events(_, let bytes) = try await api.openArena(prompt: "Salut", modelIds: ["a", "or:b/c"], effort: "low") else { Issue.record("SSE expected"); return }125        var parser = SSEParser(); var acc = ArenaAccumulator(modelIds: ["claude-haiku-5-5", "or:google/gemini-4-neon"])126        for try await chunk in bytes { for e in parser.feed(chunk) { acc.apply(ArenaEvent(e)) } }127        #expect(acc.finished); #expect(acc.comparableAnswers == 2)128        let sentBody = try #require(t.requests[0].body); let sent = try #require(JSONValue.parse(sentBody))129        #expect(sent["prompt"]?.stringValue == "Salut"); #expect(sent["modelIds"]?.arrayValue?.compactMap(\.stringValue) == ["a", "or:b/c"]); #expect(sent["settings"]?["effort"]?.stringValue == "low")130131        try await api.preferArenaAnswer(comparisonId: "cmp_1", modelId: "or:b/c")132        #expect(t.requests[1].body.flatMap(JSONValue.parse) == ["id": "cmp_1", "preferredModel": "or:b/c"])133        let adopted = try await api.adoptArenaAnswer(conversationId: "conv_1", comparisonId: "cmp_1", modelId: "a"); #expect(adopted == "msg_9")134        #expect(t.requests[2].body.flatMap(JSONValue.parse) == ["comparisonId": "cmp_1", "modelId": "a"])135        let file = try await api.exportArena(comparisonId: "cmp_1", format: .markdown)136        #expect(file.filename == "arene-2026-11-19.md"); #expect(file.mime == "text/markdown"); #expect(t.requests[3].query?.contains("format=md") == true)137        let stored = try await api.arenaComparison("cmp_r3qcjmvv6yj5uu9r"); #expect(stored?.columns.count == 2)138    }139}140141@Suite struct CostIntelligenceTests {142    @Test func decodesTheRealReport() throws {143        let u = try JSONDecoder.api().decode(UsageReport.self, from: Data(ArenaFixtures.usage.utf8))144        #expect(u.totals.requests == 965); #expect(u.byRoute.count == 4)145        let or = try #require(u.byRoute.first { $0.route == "openrouter" })146        #expect(or.provider == "openrouter"); #expect(or.costExact == 0.064968); #expect(or.requests == 7)147        #expect(u.byRoute.filter { $0.route == "direct" }.map(\.provider) == ["anthropic", "openai", "xai"])148        #expect(u.byConversation.count == 6); #expect(u.byConversation[0].title == "Histoire des phares à travers les âges"); #expect(u.byConversation[0].lastAt != nil)149        #expect(u.costBasis?.exact == 0.064968); #expect(u.costBasis?.estimated == 0.634193); #expect(u.costBasis?.note?.hasPrefix("« exact »") == true)150        #expect(u.byProject.first?.project == "—"); #expect(u.byProject.first?.projectId == nil); #expect(u.monthly.count == 1); #expect(u.monthly[0].cost == 0.699161)151        #expect(u.range?.from != nil)152    }153154    @Test func olderServerAndBrokenRowsNeverFail() throws {155        // 0.3 server: none of the new keys156        let old = #"{"totals":{"requests":1,"cost":0.1},"daily":[],"byModel":[],"byKind":[],"spending":{"today":0,"month":0,"limits":{"dailyLimitUsd":10,"monthlyLimitUsd":150,"alertAtPercent":80}}}"#157        let u = try JSONDecoder.api().decode(UsageReport.self, from: Data(old.utf8))158        #expect(u.byRoute.isEmpty); #expect(u.byConversation.isEmpty); #expect(u.costBasis == nil); #expect(u.monthly.isEmpty); #expect(u.range == nil)159        // new keys with unreadable rows and wrong types: the readable rows survive160        let odd = #"{"totals":{"requests":1,"cost":0.1},"daily":[],"byModel":[],"byKind":[],"spending":{"today":0,"month":0,"limits":{"dailyLimitUsd":10,"monthlyLimitUsd":150,"alertAtPercent":80}},"byRoute":[{"provider":"openrouter","cost":"0.5","cost_exact":"0.5","requests":"3"},42],"byConversation":[{"title":"sans id"},{"conversation_id":"c1","cost":1}],"costBasis":"n/a","monthly":{"x":1},"byProject":null}"#161        let v = try JSONDecoder.api().decode(UsageReport.self, from: Data(odd.utf8))162        #expect(v.byRoute.count == 1); #expect(v.byRoute[0].route == "openrouter"); #expect(v.byRoute[0].cost == 0.5); #expect(v.byRoute[0].requests == 3)163        #expect(v.byConversation.map(\.conversationId) == ["c1"]); #expect(v.costBasis == nil); #expect(v.monthly.isEmpty); #expect(v.byProject.isEmpty)164    }165}166167@Suite struct ConstellationTests {168    @Test func decodesTheRealGraph() throws {169        let g = try JSONDecoder.api().decode(Constellation.self, from: Data(ArenaFixtures.constellation.utf8))170        #expect(g.nodes.count == 6); #expect(g.links.count == 7); #expect(g.links.allSatisfy { $0.kind == "topic" })171        #expect(g.basis["topic"] == "Proximité sémantique : similarité cosinus entre les plongements moyens des messages (seuil 0,55, 3 voisins au plus).")172        #expect(g.basis["branch"] == "Lien de bifurcation enregistré."); #expect(g.basis["project"] == "Conversations du même projet, par ordre chronologique.")173        let n = try #require(g.nodes.first { $0.id == "conv_757ydbh4j2f7lqk6" })174        #expect(n.title == "Essai mobile 375"); #expect(n.favorite); #expect(n.messages == 2); #expect(n.costUsd == 0.000131); #expect(n.projectColor == nil); #expect(n.lastAt != nil)175        #expect(g.neighbours(of: "conv_757ydbh4j2f7lqk6") == ["conv_ch5pacgypxiwsuma"])176    }177178    @Test func dropsDanglingLinksAndUnreadableRows() throws {179        let json = ##"{"nodes":[{"id":"a","title":"A","messages":"3","costUsd":"0.5","projectId":"p","projectColor":"#5A45E0"},{"title":"sans id"},{"id":"b"},{"id":"a","title":"doublon"}],"links":[{"source":"a","target":"b","kind":"branch","weight":1},{"source":"a","target":"zzz","kind":"topic","weight":0.9},{"source":"a","target":"a"},{"source":"b","target":"a","kind":"wormhole"}],"projects":[{"id":"p","name":"Projet","color":"#5A45E0","count":"1"}],"basis":{"topic":"t","extra":7}}"##180        let g = try JSONDecoder.api().decode(Constellation.self, from: Data(json.utf8))181        #expect(g.nodes.map(\.id) == ["a", "b"]); #expect(g.nodes[0].messages == 3); #expect(g.nodes[0].costUsd == 0.5); #expect(g.nodes[1].title.isEmpty)182        #expect(g.links.map(\.kind) == ["branch", "wormhole"]); #expect(g.links[1].weight == 1)183        #expect(g.projects.first?.count == 1); #expect(g.basis == ["topic": "t"])184        let empty = try JSONDecoder.api().decode(Constellation.self, from: Data("{}".utf8))185        #expect(empty.nodes.isEmpty); #expect(empty.links.isEmpty); #expect(empty.basis.isEmpty)186    }187188    @Test func endpointClampsTheLimit() async throws {189        let t = MockTransport(); let api = makeClient(t)190        t.on("GET", "/api/constellation", .json(200, ArenaFixtures.constellation))191        _ = try await api.constellation(limit: 5000, projectId: "proj_1"); _ = try await api.constellation(limit: 1)192        #expect(t.requests[0].query?.contains("limit=300") == true); #expect(t.requests[0].query?.contains("project=proj_1") == true)193        #expect(t.requests[1].query?.contains("limit=10") == true); #expect(t.requests[1].query?.contains("project") == false)194    }195}196197@Suite struct ForceLayoutTests {198    /// A ring of clusters: `clusters` groups of `size` fully linked nodes, consecutive groups joined by one weak edge.199    private func graph(clusters: Int, size: Int) -> (radii: [Double], edges: [ForceLayout.Edge]) {200        var edges: [ForceLayout.Edge] = []201        for c in 0..<clusters {202            let base = c * size203            for i in 0..<size { for j in (i + 1)..<size { edges.append(.init(base + i, base + j, strength: 0.9)) } }204            edges.append(.init(base, ((c + 1) % clusters) * size, strength: 0.2))205        }206        return ((0..<(clusters * size)).map { 6 + Double($0 % 5) * 2 }, edges)207    }208    private func distance(_ a: ForceLayout.Point, _ b: ForceLayout.Point) -> Double { ((a.x - b.x) * (a.x - b.x) + (a.y - b.y) * (a.y - b.y)).squareRoot() }209210    @Test func deterministicForASeed() {211        let g = graph(clusters: 5, size: 6)212        var c = ForceLayout.Configuration(); c.seed = 42213        let a = ForceLayout.run(radii: g.radii, edges: g.edges, width: 900, height: 600, configuration: c)214        let b = ForceLayout.run(radii: g.radii, edges: g.edges, width: 900, height: 600, configuration: c)215        #expect(a == b)216        c.seed = 43217        #expect(ForceLayout.run(radii: g.radii, edges: g.edges, width: 900, height: 600, configuration: c).positions != a.positions)218    }219220    @Test func convergesFiniteAndInBounds() {221        for (clusters, size, w, h) in [(5, 6, 900.0, 600.0), (1, 2, 300.0, 200.0), (20, 8, 1400.0, 900.0), (12, 1, 500.0, 500.0)] {222            let g = graph(clusters: clusters, size: size)223            let r = ForceLayout.run(radii: g.radii, edges: g.edges, width: w, height: h)224            #expect(r.positions.count == g.radii.count)225            #expect(r.residual < 1, "settled: last move \(r.residual) pt for \(clusters)×\(size)")226            for (i, p) in r.positions.enumerated() {227                #expect(p.x.isFinite && p.y.isFinite)228                #expect(p.x >= g.radii[i] && p.x <= w - g.radii[i] && p.y >= g.radii[i] && p.y <= h - g.radii[i], "node \(i) inside the bounds")229            }230        }231    }232233    @Test func linkedNodesEndCloserThanUnlinkedOnes() {234        let g = graph(clusters: 4, size: 6)235        let r = ForceLayout.run(radii: g.radii, edges: g.edges, width: 1000, height: 700)236        var inside = 0.0, insideCount = 0.0, across = 0.0, acrossCount = 0.0237        for i in 0..<24 { for j in (i + 1)..<24 {238            let d = distance(r.positions[i], r.positions[j])239            if i / 6 == j / 6 { inside += d; insideCount += 1 } else { across += d; acrossCount += 1 }240        } }241        #expect(inside / insideCount < 0.6 * (across / acrossCount), "clusters are visible")242        // nodes never sit on top of each other243        for i in 0..<24 { for j in (i + 1)..<24 { #expect(distance(r.positions[i], r.positions[j]) > 4) } }244    }245246    @Test func degenerateInputs() {247        #expect(ForceLayout.run(radii: [], edges: [], width: 100, height: 100).positions.isEmpty)248        #expect(ForceLayout.run(radii: [10], edges: [.init(0, 0)], width: 200, height: 100).positions == [ForceLayout.Point(x: 100, y: 50)])249        // invalid edges, NaN inputs, zero size: still finite250        let odd = ForceLayout.run(radii: [5, .nan, -3, 8], edges: [.init(0, 9), .init(-1, 2), .init(1, 1), .init(0, 1, strength: .nan), .init(2, 3, strength: 7)], width: 0, height: .infinity)251        #expect(odd.positions.count == 4); #expect(odd.positions.allSatisfy { $0.x.isFinite && $0.y.isFinite })252        var c = ForceLayout.Configuration(); c.iterations = 0253        let one = ForceLayout.run(radii: [4, 4, 4], edges: [], width: 300, height: 300, configuration: c)254        #expect(one.iterations == 1); #expect(one.positions.allSatisfy { $0.x.isFinite && $0.y.isFinite })255    }256257    @Test func threeHundredNodesStaysFast() {258        let g = graph(clusters: 30, size: 10)259        // fewer iterations than the default: this test shares the machine with the other suites' timing tests260        var c = ForceLayout.Configuration(); c.iterations = 100261        let clock = ContinuousClock(); var result: ForceLayout.Result?262        let elapsed = clock.measure { result = ForceLayout.run(radii: g.radii, edges: g.edges, width: 2000, height: 1400, configuration: c) }263        #expect(result?.positions.count == 300)264        #expect(elapsed < .seconds(8), "300 nodes laid out in \(elapsed) (debug build)")265    }266}267
Files main
Menu

Theme