Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/AppTests/ExplorerTests.swift3// Description: Hosted tests of the Model Explorer and of Settings → Fournisseurs: view-model filtering on a real trimmed catalogue, compare tray and table, conversation hand-off, key flows on a scripted transport, and off-screen renders (light + dark).4import SwiftUI5import XCTest6@testable import SPChat7@testable import SPChatCore89@MainActor final class ExplorerTests: XCTestCase {10 /// Environment whose catalogue is the real trimmed one (34 rows) — registered before the generic fixture so that it wins.11 private func makeEnv(_ t: MockTransport = MockTransport()) async -> (AppEnvironment, MockTransport) {12 t.on("GET", "/api/models", .json(200, ExplorerFixtures.catalog))13 t.on("GET", "/api/providers", .json(200, ExplorerFixtures.providers))14 let env = Fixture.env(t)15 await env.catalog.load()16 return (env, t)17 }18 private func decode<T: Decodable>(_ type: T.Type, _ json: String) throws -> T { try JSONDecoder.api().decode(type, from: Data(json.utf8)) }1920 // MARK: View model2122 func testExplorerModelSearchesAndFiltersTheWholeCatalogue() async {23 let (env, _) = await makeEnv()24 XCTAssertEqual(env.catalog.models.models.count, 34)25 let m = ExplorerModel(); m.setIndex(env.catalog.fullIndex)26 XCTAssertEqual(m.results.count, 29); XCTAssertEqual(m.universeCount, 29)27 XCTAssertEqual(m.results.first?.id, "claude-haiku-5-5", "favourites first")28 XCTAssertEqual(m.makers.first?.slug, "anthropic"); XCTAssertEqual(m.makers.first?.count, 4); XCTAssertEqual(m.makers.first { $0.slug == "openai" }?.count, 3, "only the conversational OpenAI rows without « tous les types »")29 XCTAssertFalse(m.results.contains { $0.kind == "image" })3031 m.query.includeNonChat = true32 XCTAssertEqual(m.results.count, 34); XCTAssertEqual(m.universeCount, 34); XCTAssertEqual(m.makers.first?.slug, "openai"); XCTAssertEqual(m.makers.first?.count, 8)33 XCTAssertTrue(m.results.contains { $0.kind == "embedding" })34 m.query.includeNonChat = false3536 m.query.text = "gemini"37 XCTAssertEqual(Set(m.results.map(\.id)), ["or:google/gemini-4-neon", "or:google/gemini-4-carbon"]); XCTAssertEqual(m.universeCount, 29, "the denominator ignores search and filters")38 m.query.text = ""39 m.toggle(maker: "anthropic"); m.toggle(route: "openrouter")40 XCTAssertEqual(m.results.map(\.id), ["or:anthropic/claude-haiku-5.5"]); XCTAssertEqual(m.query.activeFilterCount, 2)41 m.toggle(route: "openrouter"); m.toggle(maker: "anthropic"); m.toggle(maker: "google"); m.toggle(maker: "meta")42 XCTAssertEqual(m.results.count, 3)43 m.toggle(category: "coding"); m.toggle(capability: "audioInput")44 XCTAssertTrue(m.results.allSatisfy { $0.categories.contains("coding") && $0.has("audioInput") }); XCTAssertFalse(m.results.isEmpty)45 m.query.text = "zzz"46 XCTAssertTrue(m.results.isEmpty)47 m.clearFilters()48 XCTAssertEqual(m.query.text, "zzz"); XCTAssertFalse(m.query.hasFilters)49 m.query.makers = ["google"]; m.clearAll()50 XCTAssertEqual(m.results.count, 29); XCTAssertTrue(m.query.text.isEmpty)5152 m.query.freeOnly = true53 XCTAssertEqual(m.results.count, 2); XCTAssertTrue(m.results.allSatisfy(\.isFree))54 m.query.freeOnly = false; m.query.openWeightOnly = true; m.query.minContext = 1_000_00055 XCTAssertTrue(m.results.allSatisfy { $0.isOpenWeight && ($0.contextWindow ?? 0) >= 1_000_000 }); XCTAssertFalse(m.results.isEmpty)56 m.clearFilters(); m.query.favoritesOnly = true57 XCTAssertEqual(Set(m.results.map(\.id)), ["claude-haiku-5-5", "or:google/gemini-4-neon"])58 }5960 func testRoutersWithAVariablePriceNeverPassAPriceCeilingAndSortLast() async {61 let (env, _) = await makeEnv()62 let m = ExplorerModel(); m.setIndex(env.catalog.fullIndex)63 let routers = m.results.filter(\.hasVariablePrice).map(\.id)64 XCTAssertEqual(Set(routers), ["or:openrouter/auto", "or:typesafe/jev-router"])65 XCTAssertNil(env.catalog.model("or:openrouter/auto")?.inputPrice, "the negative sentinel is not a price"); XCTAssertEqual(env.catalog.model("or:openrouter/auto")?.factsLine.contains("prix variable"), true)66 m.query.maxInputPrice = 0.567 XCTAssertFalse(m.results.isEmpty); XCTAssertTrue(m.results.allSatisfy { !$0.hasVariablePrice && ($0.inputPrice ?? 9) <= 0.5 })68 m.query.maxInputPrice = nil69 for sort in [ModelSort.priceAscending, .priceDescending] {70 m.query.sort = sort71 XCTAssertEqual(Set(m.results.suffix(2).map(\.id)), Set(routers), "\(sort)")72 let priced = m.results.dropLast(2).compactMap(\.blendedPrice)73 XCTAssertEqual(priced, sort == .priceAscending ? priced.sorted() : priced.sorted(by: >), "\(sort)")74 }75 m.query.sort = .context76 XCTAssertEqual(m.results.first?.id, "or:openrouter/auto")77 }7879 func testCompareTrayHoldsTwoToFourModels() async {80 let (env, _) = await makeEnv()81 let m = ExplorerModel(); m.setIndex(env.catalog.fullIndex)82 XCTAssertFalse(m.canCompare)83 XCTAssertTrue(m.toggleCompare("claude-haiku-5-5")); XCTAssertFalse(m.canCompare)84 XCTAssertTrue(m.toggleCompare("or:google/gemini-4-neon")); XCTAssertTrue(m.canCompare)85 XCTAssertTrue(m.toggleCompare("claude-opus-5-5")); XCTAssertTrue(m.toggleCompare("or:meta/muse-ember")); XCTAssertTrue(m.compareIsFull)86 XCTAssertFalse(m.toggleCompare("claude-fable-5"), "a fifth model is refused"); XCTAssertEqual(m.compare.count, 4)87 XCTAssertTrue(m.toggleCompare("claude-opus-5-5"), "removing is always possible")88 XCTAssertEqual(m.compare, ["claude-haiku-5-5", "or:google/gemini-4-neon", "or:meta/muse-ember"], "picking order is kept")89 XCTAssertTrue(m.toggleCompare("gone-model")); m.pruneCompare { env.catalog.model($0) != nil }90 XCTAssertEqual(m.compare.count, 3)91 m.clearCompare(); XCTAssertTrue(m.compare.isEmpty)92 }9394 func testCompareTableShowsPublishedFactsSideBySide() async throws {95 let (env, _) = await makeEnv()96 let models = ["claude-haiku-5-5", "or:google/gemini-4-neon", "or:openrouter/auto"].compactMap { env.catalog.model($0) }97 let lines = CompareTable.lines(for: models, defaultModelId: "claude-haiku-5-5")98 func line(_ label: String) throws -> CompareLine { try XCTUnwrap(lines.first { $0.label == label }, label) }99 XCTAssertEqual(try line("Fabricant").values, ["Anthropic", "Google", "OpenRouter"])100 XCTAssertEqual(try line("Route").values, ["Direct", "OpenRouter", "OpenRouter"])101 XCTAssertTrue(try line("Statut").values[0].contains("défaut")); XCTAssertTrue(try line("Statut").values[1].contains("non testé"))102 XCTAssertEqual(try line("Prix d'entrée / M").values, ["0,1 $", "3 $", "variable"]); XCTAssertEqual(try line("Prix d'entrée / M").best, [0], "a variable price is never the best")103 XCTAssertEqual(try line("Fenêtre de contexte").best, [2]); XCTAssertEqual(try line("Entrée en cache / M").values.last, "—")104 XCTAssertEqual(try line("Banc local · tests").values, ["11 / 11 réussis", "non testé par le banc", "non testé par le banc"])105 XCTAssertEqual(try line("Identifiant").values, models.map(\.id))106 XCTAssertTrue(lines.allSatisfy { $0.values.count == 3 }); XCTAssertEqual(Set(lines.map(\.label)).count, lines.count)107 // a fact unknown for every model is left out; equal values have no « best »108 let same = CompareTable.lines(for: [models[0], models[0]], defaultModelId: nil)109 XCTAssertNil(same.first { $0.label == "Entrées" }); XCTAssertTrue(same.allSatisfy(\.best.isEmpty))110 }111112 func testStartingAConversationWithAModelUsesItForTheNextTurnOnly() async {113 let (env, _) = await makeEnv()114 let window = WindowModel(); window.destination = .library(.models)115 let before = window.draftKey; let preferred = env.preferences.modelId116 XCTAssertTrue(window.startConversation(withModel: "or:google/gemini-4-neon", env: env))117 XCTAssertEqual(window.destination, .chat(nil)); XCTAssertNotEqual(window.draftKey, before)118 let chat = env.chat(conversationId: nil, draftKey: window.draftKey)119 XCTAssertEqual(chat.settings.modelId, "or:google/gemini-4-neon"); XCTAssertEqual(chat.settings.routingMode, .manual); XCTAssertNil(chat.settings.presetId)120 chat.ensureModel()121 XCTAssertEqual(chat.settings.modelId, "or:google/gemini-4-neon", "a catalogue model stays selected once the catalogue is checked")122 XCTAssertEqual(chat.settings.options.payload["modelId"]?.stringValue, "or:google/gemini-4-neon")123 XCTAssertEqual(env.preferences.modelId, preferred, "the owner's default is not rewritten before a turn is sent")124 // an image model cannot start a conversation: nothing moves125 window.destination = .library(.models); let key = window.draftKey126 XCTAssertFalse(window.startConversation(withModel: "chatgpt-image-latest", env: env)); XCTAssertFalse(window.startConversation(withModel: "unknown", env: env))127 XCTAssertEqual(window.destination, .library(.models)); XCTAssertEqual(window.draftKey, key)128 }129130 func testFormattingNeverInventsAFigure() {131 XCTAssertEqual(ExplorerText.price(nil), "—"); XCTAssertEqual(ExplorerText.price(0), "0 $"); XCTAssertEqual(ExplorerText.price(-1_000_000), "variable"); XCTAssertEqual(ExplorerText.price(12.5), "12,5 $")132 XCTAssertEqual(ExplorerText.price(0.09999999999999999), "0,1 $")133 XCTAssertEqual(ExplorerText.unitPrice(0.014), "0,014 $"); XCTAssertEqual(ExplorerText.unitPrice(0), "0 $")134 XCTAssertTrue(ExplorerText.unitPrice(1.5e-07).hasSuffix(" $")); XCTAssertNotEqual(ExplorerText.unitPrice(1.5e-07), "0 $", "a tiny unit price keeps its digits")135 XCTAssertEqual(ExplorerText.pricingKeys(["output": 5, "input": 1, "notes": "x", "zeta": 2, "cachedInput": 0.1, "tiers": []]), ["input", "output", "cachedInput", "zeta"])136 XCTAssertEqual(ExplorerText.pricingLabel("cacheWrite1h"), "Écriture du cache (1 h)"); XCTAssertEqual(ExplorerText.pricingLabel("somethingNew"), "somethingNew")137 XCTAssertEqual(ExplorerText.kind("hologram"), "hologram"); XCTAssertEqual(ExplorerText.sourceLabel("artificial_analysis"), "Artificial Analysis")138 let unknown = AIModel(id: "x", provider: "openai", displayName: "X", status: "quarantined")139 XCTAssertEqual(ExplorerText.status(unknown).label, "quarantined")140 let catalog = AIModel(id: "or:a/b", provider: "openrouter", displayName: "B", status: "catalog")141 XCTAssertEqual(ExplorerText.status(catalog).label, "Catalogue OpenRouter · non testé par le banc"); XCTAssertFalse(catalog.canBeDefault)142 }143144 // MARK: Providers145146 func testProvidersModelLoadsTestsAndManagesKeysWithoutKeepingThem() async throws {147 let t = MockTransport()148 t.on("POST", "/api/providers/openrouter/test", .json(200, #"{"ok":true,"message":"Clé acceptée par OpenRouter.","info":{"label":"sk-or-v1-abc...xyz","usage":0.0693,"limit":null,"isFreeTier":false}}"#))149 t.on("POST", "/api/providers/tavily/test", .fail(.timedOut))150 t.on("PUT", "/api/providers/openrouter/key", .json(422, #"{"ok":false,"message":"OpenRouter a refusé cette clé."}"#),151 .json(403, #"{"error":"Confirme avec ta clé d'accès","code":"reauth_required"}"#), .json(200, #"{"ok":true}"#))152 t.on("DELETE", "/api/providers/openrouter/key", .json(200, #"{"removed":true,"fallback":true}"#))153 let (env, _) = await makeEnv(t)154 let reauths = ReauthCount(); env.api.reauthenticator = { reauths.increment() }155 let model = ProvidersModel(env: env)156 await model.load()157 XCTAssertEqual(model.overview?.providers.count, 5); XCTAssertNil(model.loadError)158159 await model.test("openrouter")160 XCTAssertEqual(model.tests["openrouter"]?.ok, true); XCTAssertEqual(model.tests["openrouter"]?.message, "Clé acceptée par OpenRouter.")161 let facts = ProviderTestInfo.facts(model.tests["openrouter"]?.info)162 XCTAssertEqual(facts.map(\.label), ["Consommation totale", "Palier gratuit"])163 XCTAssertFalse(facts.contains { $0.value.contains("sk-or") }, "the masked key label of the test answer is never shown")164 await model.test("tavily")165 XCTAssertEqual(model.tests["tavily"]?.ok, false); XCTAssertFalse(model.tests["tavily"]?.message.isEmpty ?? true); XCTAssertTrue(model.busy.isEmpty)166167 // refused by the provider: the sheet stays open with the provider's message168 let secret = "sk-or-test-SECRET-0000"169 let refused = await model.setKey("openrouter", key: secret)170 XCTAssertFalse(refused); XCTAssertEqual(model.keyError, "OpenRouter a refusé cette clé.")171 // accepted after the passkey confirmation: replayed once, catalogue and providers reloaded172 let providersBefore = t.requests.filter { $0.path == "/api/providers" }.count, modelsBefore = t.requests.filter { $0.path == "/api/models" }.count173 let accepted = await model.setKey("openrouter", key: secret)174 XCTAssertTrue(accepted); XCTAssertNil(model.keyError); XCTAssertEqual(reauths.value, 1); XCTAssertNil(model.tests["openrouter"], "a test result of the previous key is dropped")175 XCTAssertEqual(t.requests.filter { $0.method == "PUT" }.count, 3)176 XCTAssertEqual(t.requests.filter { $0.path == "/api/providers" }.count, providersBefore + 1); XCTAssertEqual(t.requests.filter { $0.path == "/api/models" }.count, modelsBefore + 1)177 XCTAssertFalse(env.toasts.current?.text.contains(secret) ?? false); XCTAssertEqual(env.toasts.current?.isError, false)178 // the key only ever travelled in the PUT bodies179 for r in t.requests where r.method != "PUT" { XCTAssertFalse(String(decoding: r.body ?? Data(), as: UTF8.self).contains(secret)); XCTAssertFalse((r.query ?? "").contains(secret)) }180 XCTAssertTrue(t.requests.allSatisfy { !$0.headers.values.contains { $0.contains(secret) } && !$0.path.contains(secret) })181 var dump = ""; Swift.dump(model, to: &dump)182 XCTAssertFalse(dump.contains(secret), "no property of the pane's state holds the key")183184 await model.removeKey("openrouter")185 XCTAssertEqual(t.requests.filter { $0.method == "DELETE" }.count, 1)186 XCTAssertTrue(env.toasts.current?.text.contains("clé de l'environnement du serveur prend le relais") ?? false, env.toasts.current?.text ?? "no toast")187 }188189 func testProvidersLoadFailureIsShownAndRetried() async {190 let t = MockTransport()191 t.on("GET", "/api/providers", .json(500, #"{"error":"Base indisponible"}"#), .json(200, ExplorerFixtures.providers))192 let env = Fixture.env(t)193 let model = ProvidersModel(env: env)194 await model.load()195 XCTAssertNil(model.overview); XCTAssertEqual(model.loadError, "Base indisponible")196 await model.load()197 XCTAssertEqual(model.overview?.providers.count, 5); XCTAssertNil(model.loadError)198 }199200 // MARK: Snapshots (look at the PNGs in $TMPDIR/SPChat-snapshots)201202 func testExplorerRendersLightAndDark() async {203 let (env, _) = await makeEnv()204 for dark in [false, true] {205 let window = WindowModel(); window.destination = .library(.models)206 let r = Snapshot.render(ModelsScreen(window: window).environment(env), name: "explorer-\(dark ? "dark" : "light")", size: CGSize(width: 1180, height: 760), dark: dark)207 XCTAssertGreaterThan(r.opaqueFraction, 0.95)208 }209 // filters on, search text, compare tray visible, every type shown210 let m = ExplorerModel(); m.setIndex(env.catalog.fullIndex)211 m.query.includeNonChat = true; m.query.routes = ["openrouter"]; m.query.categories = ["long-context"]; m.query.maxInputPrice = 3; m.query.sort = .priceAscending212 m.toggleCompare("or:google/gemini-4-neon"); m.toggleCompare("or:meta/muse-ember"); m.toggleCompare("claude-haiku-5-5")213 for dark in [false, true] {214 let r = Snapshot.render(ModelsScreen(window: WindowModel(), model: m).environment(env), name: "explorer-filtered-\(dark ? "dark" : "light")", size: CGSize(width: 1180, height: 760), dark: dark)215 XCTAssertGreaterThan(r.opaqueFraction, 0.95)216 }217 let all = ExplorerModel(); all.setIndex(env.catalog.fullIndex); all.query.includeNonChat = true; all.query.makers = ["openai"]218 Snapshot.render(ModelsScreen(window: WindowModel(), model: all).environment(env), name: "explorer-all-kinds-light", size: CGSize(width: 980, height: 560))219 let none = ExplorerModel(); none.setIndex(env.catalog.fullIndex); none.query.text = "introuvable"220 Snapshot.render(ModelsScreen(window: WindowModel(), model: none).environment(env), name: "explorer-empty-light", size: CGSize(width: 980, height: 560))221 print("SNAPSHOTS: \(Snapshot.directory.path)")222 }223224 func testModelDetailRendersLightAndDark() async throws {225 let (env, _) = await makeEnv()226 let explorer = ExplorerModel(); explorer.categoryRules = try decode(ProvidersOverview.self, ExplorerFixtures.providers).categoryRules227 let or = try decode(ModelDetail.self, ExplorerFixtures.detailOpenRouter), direct = try decode(ModelDetail.self, ExplorerFixtures.detailDirect)228 for dark in [false, true] {229 let a = Snapshot.render(ModelDetailView(modelId: or.model.id, window: WindowModel(), explorer: explorer, preloaded: or, close: {}).environment(env).frame(width: 800, height: 1500),230 name: "detail-openrouter-\(dark ? "dark" : "light")", size: CGSize(width: 800, height: 1500), dark: dark)231 XCTAssertGreaterThan(a.opaqueFraction, 0.95)232 Snapshot.render(ModelDetailView(modelId: direct.model.id, window: WindowModel(), explorer: explorer, preloaded: direct, close: {}).environment(env).frame(width: 800, height: 1250),233 name: "detail-direct-\(dark ? "dark" : "light")", size: CGSize(width: 800, height: 1250), dark: dark)234 }235 // without the live payload the sheet still shows the catalogue facts (offline)236 Snapshot.render(ModelDetailView(modelId: "or:tencent/hy4-flash", window: WindowModel(), explorer: explorer, close: {}).environment(env).frame(width: 800, height: 760),237 name: "detail-catalogue-only-light", size: CGSize(width: 800, height: 760))238 }239240 func testCompareTableAndProvidersPaneRenderLightAndDark() async throws {241 let (env, _) = await makeEnv()242 let models = ["claude-haiku-5-5", "or:google/gemini-4-neon", "or:tencent/hy4-flash", "or:openrouter/auto"].compactMap { env.catalog.model($0) }243 XCTAssertEqual(models.count, 4)244 let providers = ProvidersModel(env: env)245 await providers.load()246 for dark in [false, true] {247 let c = Snapshot.render(ModelCompareView(models: models, defaultModelId: "claude-haiku-5-5", close: {}, arena: {}).environment(env), name: "compare-\(dark ? "dark" : "light")", dark: dark)248 XCTAssertGreaterThan(c.size.width, 900); XCTAssertGreaterThan(c.opaqueFraction, 0.95)249 let p = Snapshot.render(ProvidersPane(model: providers, setKey: { _ in }, revoke: { _ in }).environment(env).frame(width: 640, height: 900), name: "providers-\(dark ? "dark" : "light")", size: CGSize(width: 640, height: 900), dark: dark)250 XCTAssertGreaterThan(p.opaqueFraction, 0.95)251 }252 let two = Snapshot.render(ModelCompareView(models: Array(models.prefix(2)), close: {}, arena: {}).environment(env), name: "compare-two-light")253 XCTAssertLessThan(two.size.width, 700)254 let key = try XCTUnwrap(providers.overview?.provider("openrouter"))255 Snapshot.render(ProviderKeySheet(provider: key, model: providers, close: {}).environment(env).background(Color(nsColor: .windowBackgroundColor)), name: "provider-key-sheet-light")256 }257}258259private final class ReauthCount: @unchecked Sendable { private let l = NSLock(); private var v = 0; func increment() { l.withLock { v += 1 } }; var value: Int { l.withLock { v } } }260