SPB Git

Repository Public

ExplorerTests.swift 19.8 KB 260 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/AppTests/ExplorerTests.swift3// Description: Hosted tests of the Model Explorer and of Settings → Fournisseurs: view-model filtering on a real trimmed catalogue, compare tray and table, conversation hand-off, key flows on a scripted transport, and off-screen renders (light + dark).4import SwiftUI5import XCTest6@testable import SPChat7@testable import SPChatCore89@MainActor final class ExplorerTests: XCTestCase {10    /// Environment whose catalogue is the real trimmed one (34 rows) — registered before the generic fixture so that it wins.11    private func makeEnv(_ t: MockTransport = MockTransport()) async -> (AppEnvironment, MockTransport) {12        t.on("GET", "/api/models", .json(200, ExplorerFixtures.catalog))13        t.on("GET", "/api/providers", .json(200, ExplorerFixtures.providers))14        let env = Fixture.env(t)15        await env.catalog.load()16        return (env, t)17    }18    private func decode<T: Decodable>(_ type: T.Type, _ json: String) throws -> T { try JSONDecoder.api().decode(type, from: Data(json.utf8)) }1920    // MARK: View model2122    func testExplorerModelSearchesAndFiltersTheWholeCatalogue() async {23        let (env, _) = await makeEnv()24        XCTAssertEqual(env.catalog.models.models.count, 34)25        let m = ExplorerModel(); m.setIndex(env.catalog.fullIndex)26        XCTAssertEqual(m.results.count, 29); XCTAssertEqual(m.universeCount, 29)27        XCTAssertEqual(m.results.first?.id, "claude-haiku-5-5", "favourites first")28        XCTAssertEqual(m.makers.first?.slug, "anthropic"); XCTAssertEqual(m.makers.first?.count, 4); XCTAssertEqual(m.makers.first { $0.slug == "openai" }?.count, 3, "only the conversational OpenAI rows without « tous les types »")29        XCTAssertFalse(m.results.contains { $0.kind == "image" })3031        m.query.includeNonChat = true32        XCTAssertEqual(m.results.count, 34); XCTAssertEqual(m.universeCount, 34); XCTAssertEqual(m.makers.first?.slug, "openai"); XCTAssertEqual(m.makers.first?.count, 8)33        XCTAssertTrue(m.results.contains { $0.kind == "embedding" })34        m.query.includeNonChat = false3536        m.query.text = "gemini"37        XCTAssertEqual(Set(m.results.map(\.id)), ["or:google/gemini-4-neon", "or:google/gemini-4-carbon"]); XCTAssertEqual(m.universeCount, 29, "the denominator ignores search and filters")38        m.query.text = ""39        m.toggle(maker: "anthropic"); m.toggle(route: "openrouter")40        XCTAssertEqual(m.results.map(\.id), ["or:anthropic/claude-haiku-5.5"]); XCTAssertEqual(m.query.activeFilterCount, 2)41        m.toggle(route: "openrouter"); m.toggle(maker: "anthropic"); m.toggle(maker: "google"); m.toggle(maker: "meta")42        XCTAssertEqual(m.results.count, 3)43        m.toggle(category: "coding"); m.toggle(capability: "audioInput")44        XCTAssertTrue(m.results.allSatisfy { $0.categories.contains("coding") && $0.has("audioInput") }); XCTAssertFalse(m.results.isEmpty)45        m.query.text = "zzz"46        XCTAssertTrue(m.results.isEmpty)47        m.clearFilters()48        XCTAssertEqual(m.query.text, "zzz"); XCTAssertFalse(m.query.hasFilters)49        m.query.makers = ["google"]; m.clearAll()50        XCTAssertEqual(m.results.count, 29); XCTAssertTrue(m.query.text.isEmpty)5152        m.query.freeOnly = true53        XCTAssertEqual(m.results.count, 2); XCTAssertTrue(m.results.allSatisfy(\.isFree))54        m.query.freeOnly = false; m.query.openWeightOnly = true; m.query.minContext = 1_000_00055        XCTAssertTrue(m.results.allSatisfy { $0.isOpenWeight && ($0.contextWindow ?? 0) >= 1_000_000 }); XCTAssertFalse(m.results.isEmpty)56        m.clearFilters(); m.query.favoritesOnly = true57        XCTAssertEqual(Set(m.results.map(\.id)), ["claude-haiku-5-5", "or:google/gemini-4-neon"])58    }5960    func testRoutersWithAVariablePriceNeverPassAPriceCeilingAndSortLast() async {61        let (env, _) = await makeEnv()62        let m = ExplorerModel(); m.setIndex(env.catalog.fullIndex)63        let routers = m.results.filter(\.hasVariablePrice).map(\.id)64        XCTAssertEqual(Set(routers), ["or:openrouter/auto", "or:typesafe/jev-router"])65        XCTAssertNil(env.catalog.model("or:openrouter/auto")?.inputPrice, "the negative sentinel is not a price"); XCTAssertEqual(env.catalog.model("or:openrouter/auto")?.factsLine.contains("prix variable"), true)66        m.query.maxInputPrice = 0.567        XCTAssertFalse(m.results.isEmpty); XCTAssertTrue(m.results.allSatisfy { !$0.hasVariablePrice && ($0.inputPrice ?? 9) <= 0.5 })68        m.query.maxInputPrice = nil69        for sort in [ModelSort.priceAscending, .priceDescending] {70            m.query.sort = sort71            XCTAssertEqual(Set(m.results.suffix(2).map(\.id)), Set(routers), "\(sort)")72            let priced = m.results.dropLast(2).compactMap(\.blendedPrice)73            XCTAssertEqual(priced, sort == .priceAscending ? priced.sorted() : priced.sorted(by: >), "\(sort)")74        }75        m.query.sort = .context76        XCTAssertEqual(m.results.first?.id, "or:openrouter/auto")77    }7879    func testCompareTrayHoldsTwoToFourModels() async {80        let (env, _) = await makeEnv()81        let m = ExplorerModel(); m.setIndex(env.catalog.fullIndex)82        XCTAssertFalse(m.canCompare)83        XCTAssertTrue(m.toggleCompare("claude-haiku-5-5")); XCTAssertFalse(m.canCompare)84        XCTAssertTrue(m.toggleCompare("or:google/gemini-4-neon")); XCTAssertTrue(m.canCompare)85        XCTAssertTrue(m.toggleCompare("claude-opus-5-5")); XCTAssertTrue(m.toggleCompare("or:meta/muse-ember")); XCTAssertTrue(m.compareIsFull)86        XCTAssertFalse(m.toggleCompare("claude-fable-5"), "a fifth model is refused"); XCTAssertEqual(m.compare.count, 4)87        XCTAssertTrue(m.toggleCompare("claude-opus-5-5"), "removing is always possible")88        XCTAssertEqual(m.compare, ["claude-haiku-5-5", "or:google/gemini-4-neon", "or:meta/muse-ember"], "picking order is kept")89        XCTAssertTrue(m.toggleCompare("gone-model")); m.pruneCompare { env.catalog.model($0) != nil }90        XCTAssertEqual(m.compare.count, 3)91        m.clearCompare(); XCTAssertTrue(m.compare.isEmpty)92    }9394    func testCompareTableShowsPublishedFactsSideBySide() async throws {95        let (env, _) = await makeEnv()96        let models = ["claude-haiku-5-5", "or:google/gemini-4-neon", "or:openrouter/auto"].compactMap { env.catalog.model($0) }97        let lines = CompareTable.lines(for: models, defaultModelId: "claude-haiku-5-5")98        func line(_ label: String) throws -> CompareLine { try XCTUnwrap(lines.first { $0.label == label }, label) }99        XCTAssertEqual(try line("Fabricant").values, ["Anthropic", "Google", "OpenRouter"])100        XCTAssertEqual(try line("Route").values, ["Direct", "OpenRouter", "OpenRouter"])101        XCTAssertTrue(try line("Statut").values[0].contains("défaut")); XCTAssertTrue(try line("Statut").values[1].contains("non testé"))102        XCTAssertEqual(try line("Prix d'entrée / M").values, ["0,1 $", "3 $", "variable"]); XCTAssertEqual(try line("Prix d'entrée / M").best, [0], "a variable price is never the best")103        XCTAssertEqual(try line("Fenêtre de contexte").best, [2]); XCTAssertEqual(try line("Entrée en cache / M").values.last, "—")104        XCTAssertEqual(try line("Banc local · tests").values, ["11 / 11 réussis", "non testé par le banc", "non testé par le banc"])105        XCTAssertEqual(try line("Identifiant").values, models.map(\.id))106        XCTAssertTrue(lines.allSatisfy { $0.values.count == 3 }); XCTAssertEqual(Set(lines.map(\.label)).count, lines.count)107        // a fact unknown for every model is left out; equal values have no « best »108        let same = CompareTable.lines(for: [models[0], models[0]], defaultModelId: nil)109        XCTAssertNil(same.first { $0.label == "Entrées" }); XCTAssertTrue(same.allSatisfy(\.best.isEmpty))110    }111112    func testStartingAConversationWithAModelUsesItForTheNextTurnOnly() async {113        let (env, _) = await makeEnv()114        let window = WindowModel(); window.destination = .library(.models)115        let before = window.draftKey; let preferred = env.preferences.modelId116        XCTAssertTrue(window.startConversation(withModel: "or:google/gemini-4-neon", env: env))117        XCTAssertEqual(window.destination, .chat(nil)); XCTAssertNotEqual(window.draftKey, before)118        let chat = env.chat(conversationId: nil, draftKey: window.draftKey)119        XCTAssertEqual(chat.settings.modelId, "or:google/gemini-4-neon"); XCTAssertEqual(chat.settings.routingMode, .manual); XCTAssertNil(chat.settings.presetId)120        chat.ensureModel()121        XCTAssertEqual(chat.settings.modelId, "or:google/gemini-4-neon", "a catalogue model stays selected once the catalogue is checked")122        XCTAssertEqual(chat.settings.options.payload["modelId"]?.stringValue, "or:google/gemini-4-neon")123        XCTAssertEqual(env.preferences.modelId, preferred, "the owner's default is not rewritten before a turn is sent")124        // an image model cannot start a conversation: nothing moves125        window.destination = .library(.models); let key = window.draftKey126        XCTAssertFalse(window.startConversation(withModel: "chatgpt-image-latest", env: env)); XCTAssertFalse(window.startConversation(withModel: "unknown", env: env))127        XCTAssertEqual(window.destination, .library(.models)); XCTAssertEqual(window.draftKey, key)128    }129130    func testFormattingNeverInventsAFigure() {131        XCTAssertEqual(ExplorerText.price(nil), "—"); XCTAssertEqual(ExplorerText.price(0), "0 $"); XCTAssertEqual(ExplorerText.price(-1_000_000), "variable"); XCTAssertEqual(ExplorerText.price(12.5), "12,5 $")132        XCTAssertEqual(ExplorerText.price(0.09999999999999999), "0,1 $")133        XCTAssertEqual(ExplorerText.unitPrice(0.014), "0,014 $"); XCTAssertEqual(ExplorerText.unitPrice(0), "0 $")134        XCTAssertTrue(ExplorerText.unitPrice(1.5e-07).hasSuffix(" $")); XCTAssertNotEqual(ExplorerText.unitPrice(1.5e-07), "0 $", "a tiny unit price keeps its digits")135        XCTAssertEqual(ExplorerText.pricingKeys(["output": 5, "input": 1, "notes": "x", "zeta": 2, "cachedInput": 0.1, "tiers": []]), ["input", "output", "cachedInput", "zeta"])136        XCTAssertEqual(ExplorerText.pricingLabel("cacheWrite1h"), "Écriture du cache (1 h)"); XCTAssertEqual(ExplorerText.pricingLabel("somethingNew"), "somethingNew")137        XCTAssertEqual(ExplorerText.kind("hologram"), "hologram"); XCTAssertEqual(ExplorerText.sourceLabel("artificial_analysis"), "Artificial Analysis")138        let unknown = AIModel(id: "x", provider: "openai", displayName: "X", status: "quarantined")139        XCTAssertEqual(ExplorerText.status(unknown).label, "quarantined")140        let catalog = AIModel(id: "or:a/b", provider: "openrouter", displayName: "B", status: "catalog")141        XCTAssertEqual(ExplorerText.status(catalog).label, "Catalogue OpenRouter · non testé par le banc"); XCTAssertFalse(catalog.canBeDefault)142    }143144    // MARK: Providers145146    func testProvidersModelLoadsTestsAndManagesKeysWithoutKeepingThem() async throws {147        let t = MockTransport()148        t.on("POST", "/api/providers/openrouter/test", .json(200, #"{"ok":true,"message":"Clé acceptée par OpenRouter.","info":{"label":"sk-or-v1-abc...xyz","usage":0.0693,"limit":null,"isFreeTier":false}}"#))149        t.on("POST", "/api/providers/tavily/test", .fail(.timedOut))150        t.on("PUT", "/api/providers/openrouter/key", .json(422, #"{"ok":false,"message":"OpenRouter a refusé cette clé."}"#),151             .json(403, #"{"error":"Confirme avec ta clé d'accès","code":"reauth_required"}"#), .json(200, #"{"ok":true}"#))152        t.on("DELETE", "/api/providers/openrouter/key", .json(200, #"{"removed":true,"fallback":true}"#))153        let (env, _) = await makeEnv(t)154        let reauths = ReauthCount(); env.api.reauthenticator = { reauths.increment() }155        let model = ProvidersModel(env: env)156        await model.load()157        XCTAssertEqual(model.overview?.providers.count, 5); XCTAssertNil(model.loadError)158159        await model.test("openrouter")160        XCTAssertEqual(model.tests["openrouter"]?.ok, true); XCTAssertEqual(model.tests["openrouter"]?.message, "Clé acceptée par OpenRouter.")161        let facts = ProviderTestInfo.facts(model.tests["openrouter"]?.info)162        XCTAssertEqual(facts.map(\.label), ["Consommation totale", "Palier gratuit"])163        XCTAssertFalse(facts.contains { $0.value.contains("sk-or") }, "the masked key label of the test answer is never shown")164        await model.test("tavily")165        XCTAssertEqual(model.tests["tavily"]?.ok, false); XCTAssertFalse(model.tests["tavily"]?.message.isEmpty ?? true); XCTAssertTrue(model.busy.isEmpty)166167        // refused by the provider: the sheet stays open with the provider's message168        let secret = "sk-or-test-SECRET-0000"169        let refused = await model.setKey("openrouter", key: secret)170        XCTAssertFalse(refused); XCTAssertEqual(model.keyError, "OpenRouter a refusé cette clé.")171        // accepted after the passkey confirmation: replayed once, catalogue and providers reloaded172        let providersBefore = t.requests.filter { $0.path == "/api/providers" }.count, modelsBefore = t.requests.filter { $0.path == "/api/models" }.count173        let accepted = await model.setKey("openrouter", key: secret)174        XCTAssertTrue(accepted); XCTAssertNil(model.keyError); XCTAssertEqual(reauths.value, 1); XCTAssertNil(model.tests["openrouter"], "a test result of the previous key is dropped")175        XCTAssertEqual(t.requests.filter { $0.method == "PUT" }.count, 3)176        XCTAssertEqual(t.requests.filter { $0.path == "/api/providers" }.count, providersBefore + 1); XCTAssertEqual(t.requests.filter { $0.path == "/api/models" }.count, modelsBefore + 1)177        XCTAssertFalse(env.toasts.current?.text.contains(secret) ?? false); XCTAssertEqual(env.toasts.current?.isError, false)178        // the key only ever travelled in the PUT bodies179        for r in t.requests where r.method != "PUT" { XCTAssertFalse(String(decoding: r.body ?? Data(), as: UTF8.self).contains(secret)); XCTAssertFalse((r.query ?? "").contains(secret)) }180        XCTAssertTrue(t.requests.allSatisfy { !$0.headers.values.contains { $0.contains(secret) } && !$0.path.contains(secret) })181        var dump = ""; Swift.dump(model, to: &dump)182        XCTAssertFalse(dump.contains(secret), "no property of the pane's state holds the key")183184        await model.removeKey("openrouter")185        XCTAssertEqual(t.requests.filter { $0.method == "DELETE" }.count, 1)186        XCTAssertTrue(env.toasts.current?.text.contains("clé de l'environnement du serveur prend le relais") ?? false, env.toasts.current?.text ?? "no toast")187    }188189    func testProvidersLoadFailureIsShownAndRetried() async {190        let t = MockTransport()191        t.on("GET", "/api/providers", .json(500, #"{"error":"Base indisponible"}"#), .json(200, ExplorerFixtures.providers))192        let env = Fixture.env(t)193        let model = ProvidersModel(env: env)194        await model.load()195        XCTAssertNil(model.overview); XCTAssertEqual(model.loadError, "Base indisponible")196        await model.load()197        XCTAssertEqual(model.overview?.providers.count, 5); XCTAssertNil(model.loadError)198    }199200    // MARK: Snapshots (look at the PNGs in $TMPDIR/SPChat-snapshots)201202    func testExplorerRendersLightAndDark() async {203        let (env, _) = await makeEnv()204        for dark in [false, true] {205            let window = WindowModel(); window.destination = .library(.models)206            let r = Snapshot.render(ModelsScreen(window: window).environment(env), name: "explorer-\(dark ? "dark" : "light")", size: CGSize(width: 1180, height: 760), dark: dark)207            XCTAssertGreaterThan(r.opaqueFraction, 0.95)208        }209        // filters on, search text, compare tray visible, every type shown210        let m = ExplorerModel(); m.setIndex(env.catalog.fullIndex)211        m.query.includeNonChat = true; m.query.routes = ["openrouter"]; m.query.categories = ["long-context"]; m.query.maxInputPrice = 3; m.query.sort = .priceAscending212        m.toggleCompare("or:google/gemini-4-neon"); m.toggleCompare("or:meta/muse-ember"); m.toggleCompare("claude-haiku-5-5")213        for dark in [false, true] {214            let r = Snapshot.render(ModelsScreen(window: WindowModel(), model: m).environment(env), name: "explorer-filtered-\(dark ? "dark" : "light")", size: CGSize(width: 1180, height: 760), dark: dark)215            XCTAssertGreaterThan(r.opaqueFraction, 0.95)216        }217        let all = ExplorerModel(); all.setIndex(env.catalog.fullIndex); all.query.includeNonChat = true; all.query.makers = ["openai"]218        Snapshot.render(ModelsScreen(window: WindowModel(), model: all).environment(env), name: "explorer-all-kinds-light", size: CGSize(width: 980, height: 560))219        let none = ExplorerModel(); none.setIndex(env.catalog.fullIndex); none.query.text = "introuvable"220        Snapshot.render(ModelsScreen(window: WindowModel(), model: none).environment(env), name: "explorer-empty-light", size: CGSize(width: 980, height: 560))221        print("SNAPSHOTS: \(Snapshot.directory.path)")222    }223224    func testModelDetailRendersLightAndDark() async throws {225        let (env, _) = await makeEnv()226        let explorer = ExplorerModel(); explorer.categoryRules = try decode(ProvidersOverview.self, ExplorerFixtures.providers).categoryRules227        let or = try decode(ModelDetail.self, ExplorerFixtures.detailOpenRouter), direct = try decode(ModelDetail.self, ExplorerFixtures.detailDirect)228        for dark in [false, true] {229            let a = Snapshot.render(ModelDetailView(modelId: or.model.id, window: WindowModel(), explorer: explorer, preloaded: or, close: {}).environment(env).frame(width: 800, height: 1500),230                                    name: "detail-openrouter-\(dark ? "dark" : "light")", size: CGSize(width: 800, height: 1500), dark: dark)231            XCTAssertGreaterThan(a.opaqueFraction, 0.95)232            Snapshot.render(ModelDetailView(modelId: direct.model.id, window: WindowModel(), explorer: explorer, preloaded: direct, close: {}).environment(env).frame(width: 800, height: 1250),233                            name: "detail-direct-\(dark ? "dark" : "light")", size: CGSize(width: 800, height: 1250), dark: dark)234        }235        // without the live payload the sheet still shows the catalogue facts (offline)236        Snapshot.render(ModelDetailView(modelId: "or:tencent/hy4-flash", window: WindowModel(), explorer: explorer, close: {}).environment(env).frame(width: 800, height: 760),237                        name: "detail-catalogue-only-light", size: CGSize(width: 800, height: 760))238    }239240    func testCompareTableAndProvidersPaneRenderLightAndDark() async throws {241        let (env, _) = await makeEnv()242        let models = ["claude-haiku-5-5", "or:google/gemini-4-neon", "or:tencent/hy4-flash", "or:openrouter/auto"].compactMap { env.catalog.model($0) }243        XCTAssertEqual(models.count, 4)244        let providers = ProvidersModel(env: env)245        await providers.load()246        for dark in [false, true] {247            let c = Snapshot.render(ModelCompareView(models: models, defaultModelId: "claude-haiku-5-5", close: {}, arena: {}).environment(env), name: "compare-\(dark ? "dark" : "light")", dark: dark)248            XCTAssertGreaterThan(c.size.width, 900); XCTAssertGreaterThan(c.opaqueFraction, 0.95)249            let p = Snapshot.render(ProvidersPane(model: providers, setKey: { _ in }, revoke: { _ in }).environment(env).frame(width: 640, height: 900), name: "providers-\(dark ? "dark" : "light")", size: CGSize(width: 640, height: 900), dark: dark)250            XCTAssertGreaterThan(p.opaqueFraction, 0.95)251        }252        let two = Snapshot.render(ModelCompareView(models: Array(models.prefix(2)), close: {}, arena: {}).environment(env), name: "compare-two-light")253        XCTAssertLessThan(two.size.width, 700)254        let key = try XCTUnwrap(providers.overview?.provider("openrouter"))255        Snapshot.render(ProviderKeySheet(provider: key, model: providers, close: {}).environment(env).background(Color(nsColor: .windowBackgroundColor)), name: "provider-key-sheet-light")256    }257}258259private final class ReauthCount: @unchecked Sendable { private let l = NSLock(); private var v = 0; func increment() { l.withLock { v += 1 } }; var value: Int { l.withLock { v } } }260
Files main
Menu

Theme