SPB Git

Repository Public

ModelQueryTests.swift 17.6 KB 236 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/CoreTests/ModelQueryTests.swift3// Description: Catalogue query engine of the Explorer: every filter, every sort (unknown values last), search ranking, maker facet, and decoding that survives unknown values and malformed rows — on a real trimmed catalogue.4import Foundation5import Testing6@testable import SPChatCore78@Suite struct ModelQueryTests {9    private static let catalog: ModelCatalog = { (try? JSONDecoder.api().decode(ModelCatalog.self, from: Data(ExplorerFixtures.catalog.utf8))) ?? ModelCatalog() }()10    private var models: [AIModel] { Self.catalog.models }11    private var index: ModelSearchIndex { ModelSearchIndex(models) }12    private func ids(_ configure: (inout ModelQuery) -> Void) -> [String] { var q = ModelQuery(); configure(&q); return index.search(q).map(\.id) }13    private func all(_ configure: (inout ModelQuery) -> Void) -> [AIModel] { var q = ModelQuery(); q.includeNonChat = true; configure(&q); return index.search(q) }1415    /// Small hand-built rows for the orderings whose expected result must be obvious.16    private func row(_ id: String, name: String? = nil, input: Double? = nil, output: Double? = nil, context: Int? = nil, used: Date? = nil, created: Int? = nil, favorite: Bool = false, status: String = "verified") -> AIModel {17        var pricing: [String: JSONValue]?18        if input != nil || output != nil { pricing = [:]; if let input { pricing?["input"] = .number(input) }; if let output { pricing?["output"] = .number(output) } }19        return AIModel(id: id, provider: id.hasPrefix("or:") ? "openrouter" : "openai", displayName: name ?? id, status: status, contextWindow: context, pricing: pricing, favorite: favorite,20                       catalog: created.map { CatalogInfo(slug: id, created: $0) }, lastUsedAt: used)21    }2223    // MARK: Decoding2425    @Test func realCatalogueDecodes() {26        #expect(models.count == 34); #expect(Self.catalog.defaultModel == "claude-opus-5-5"); #expect(Self.catalog.dock == ["claude-haiku-5-5", "or:google/gemini-4-neon"])27        #expect(Self.catalog.openrouter?.configured == true); #expect(Self.catalog.openrouter?.catalog?.total == 444); #expect(Self.catalog.refreshedAt != nil)28        #expect(models.filter(\.isOpenRouter).count == 24); #expect(models.filter(\.isVerified).count == 10); #expect(models.filter(\.isCatalogOnly).count == 24)29        let neon = Self.catalog.model("or:google/gemini-4-neon")30        #expect(neon?.creator == "google"); #expect(neon?.logo == "google"); #expect(neon?.route == "openrouter"); #expect(neon?.catalog?.benchmarkSources == ["artificial_analysis"] || neon?.catalog?.benchmarkSources.isEmpty == false)31        #expect(Self.catalog.model("or:inclusionai/ling-3.2-flash:free")?.logo == nil); #expect(Self.catalog.model("or:inclusionai/ling-3.2-flash:free")?.isFree == true)32        // the cache encoding gives the same catalogue back (dates are compared apart: they are re-encoded to the millisecond)33        func undated(_ m: AIModel) -> AIModel { var c = m; c.lastVerifiedAt = nil; c.lastUsedAt = nil; return c }34        let back = try? JSONDecoder.api().decode(ModelCatalog.self, from: JSONEncoder.api().encode(Self.catalog))35        #expect(back?.models.map(undated) == models.map(undated)); #expect(back?.dock == Self.catalog.dock); #expect(back?.defaultModel == Self.catalog.defaultModel)36        #expect(zip(back?.models ?? [], models).allSatisfy { abs(($0.lastVerifiedAt?.timeIntervalSince1970 ?? 0) - ($1.lastVerifiedAt?.timeIntervalSince1970 ?? 0)) < 0.002 })37    }3839    @Test func unknownValuesAndMalformedRowsNeverBreakTheCatalogue() throws {40        let json = #"{"models":[{"id":"ok-1","provider":"openai","displayName":"OK","kind":"chat","status":"verified","available":true},{"displayName":"no id"},"junk",null,{"id":"future-1","provider":"hyperscaler","kind":"hologram","status":"quarantined","available":true,"route":"satellite","contextWindow":"200000","pricing":{"input":"1.5","output":null,"tiers":"n/a"},"capabilities":{"vision":"unknown","tools":true},"categories":["quantum"],"catalog":{"created":"1790000000","prices":{"image":"0.01","bad":{}}},"benchmark":"none","lastUsedAt":"yesterday"},{"id":"or:new-vendor/model-x"}],"dock":"none","openrouter":{"configured":"maybe"}}"#41        let c = try JSONDecoder.api().decode(ModelCatalog.self, from: Data(json.utf8))42        #expect(c.models.map(\.id) == ["ok-1", "future-1", "or:new-vendor/model-x"]); #expect(c.dock.isEmpty); #expect(c.openrouter?.configured == false)43        let f = c.models[1]44        #expect(f.provider == "hyperscaler"); #expect(f.status == "quarantined"); #expect(f.kind == "hologram"); #expect(f.route == "satellite"); #expect(f.contextWindow == 200_000)45        #expect(f.inputPrice == 1.5); #expect(f.outputPrice == nil); #expect(f.blendedPrice == nil); #expect(!f.has("vision")); #expect(f.has("tools")); #expect(f.categories == ["quantum"])46        #expect(f.catalog?.created == 1_790_000_000); #expect(f.catalog?.prices == ["image": 0.01]); #expect(f.benchmark == nil); #expect(f.lastUsedAt == nil)47        #expect(!f.isVerified); #expect(!f.isCatalogOnly); #expect(!f.isSelectableForChat)48        let bare = c.models[2]49        #expect(bare.route == "openrouter"); #expect(bare.creator == "new-vendor"); #expect(bare.displayName == "or:new-vendor/model-x"); #expect(bare.status == "untested"); #expect(!bare.available)50        // unknown values are searchable and filterable like any other51        var q = ModelQuery(); q.includeNonChat = true; q.routes = ["satellite"]52        #expect(c.models.applying(q).map(\.id) == ["future-1"])53        q = ModelQuery(); q.includeNonChat = true; q.categories = ["quantum"]; q.text = "hyperscaler"54        #expect(c.models.applying(q).map(\.id) == ["future-1"])55        #expect(ModelFacet.label("quantum") == "Quantum"); #expect(ModelFacet.symbol("quantum") == "tag"); #expect(ModelFacet.capabilityLabel("telepathy") == "telepathy")56    }5758    // MARK: Filters5960    @Test func defaultQueryKeepsOnlyModelsThatCanBePickedForAConversation() {61        let picked = ids { _ in }62        #expect(picked.count == 29)   // 34 rows − image, tts, stt, embedding, moderation63        #expect(!picked.contains("chatgpt-image-latest")); #expect(!picked.contains("text-embedding-3-large")); #expect(picked.contains("gpt-5-pro")); #expect(picked.contains("gpt-5-search-api"))64        #expect(all { _ in }.count == 34)65        // hidden and unavailable rows only appear with « tous les types »66        var rows = [row("a"), row("b"), row("c")]; rows[1].hidden = true; rows[2].available = false67        #expect(rows.applying(ModelQuery()).map(\.id) == ["a"])68        var q = ModelQuery(); q.includeNonChat = true69        #expect(Set(rows.applying(q).map(\.id)) == ["a", "b", "c"])70    }7172    @Test func makerFilter() {73        let google = ids { $0.makers = ["google"] }74        #expect(Set(google) == ["or:google/gemini-4-neon", "or:google/gemini-4-carbon"])75        let two = ids { $0.makers = ["anthropic", "meta"] }76        #expect(Set(two) == ["claude-haiku-5-5", "claude-opus-5-5", "claude-fable-5", "or:anthropic/claude-haiku-5.5", "or:meta/muse-ember"])77        #expect(ids { $0.makers = ["nobody"] }.isEmpty)78    }7980    @Test func routeFilter() {81        let direct = ids { $0.routes = ["direct"] }, or = ids { $0.routes = ["openrouter"] }82        #expect(direct.count == 5); #expect(direct.allSatisfy { !$0.hasPrefix("or:") })83        #expect(or.count == 24); #expect(or.allSatisfy { $0.hasPrefix("or:") })84        #expect(ids { $0.routes = ["direct", "openrouter"] }.count == 29)85        // the same model by two routes stays two rows86        let haiku = ids { $0.text = "claude haiku 5" }87        #expect(Set(haiku) == ["claude-haiku-5-5", "or:anthropic/claude-haiku-5.5"])88    }8990    @Test func capabilityFilterRequiresEveryFlag() {91        let vision = all { $0.capabilities = ["vision"] }92        #expect(!vision.isEmpty); #expect(vision.allSatisfy { $0.has("vision") })93        // « unknown » is not true94        #expect(!vision.contains { $0.id == "gpt-5-pro" })95        let both = all { $0.capabilities = ["vision", "audioInput"] }96        #expect(!both.isEmpty); #expect(both.count < vision.count); #expect(both.allSatisfy { $0.has("vision") && $0.has("audioInput") })97        #expect(all { $0.capabilities = ["imageGeneration"] }.map(\.id).contains("chatgpt-image-latest"))98        #expect(ids { $0.capabilities = ["telepathy"] }.isEmpty)99    }100101    @Test func categoryFilterRequiresEveryCategory() {102        let coding = ids { $0.categories = ["coding"] }103        #expect(coding.count == 7); #expect(coding.contains("or:tencent/hy4-flash"))104        let both = ids { $0.categories = ["coding", "research"] }105        #expect(Set(both) == ["or:tencent/hy4-flash", "or:moonshotai/kimi-k3.1-pro", "or:deepseek/deepseek-v4.1-flash-1017"])106        #expect(ids { $0.categories = ["fast"] }.allSatisfy { !$0.hasPrefix("or:") })107    }108109    @Test func priceCeilingExcludesModelsWithoutAPrice() {110        let cheap = all { $0.maxInputPrice = 0.1 }111        #expect(!cheap.isEmpty); #expect(cheap.allSatisfy { ($0.inputPrice ?? .infinity) <= 0.1 })112        #expect(!cheap.contains { $0.id == "gpt-4o-mini-transcribe" })   // no input price published113        #expect(cheap.contains { $0.id == "or:deepseek/deepseek-v4.1-flash-1017" }); #expect(!cheap.contains { $0.id == "claude-opus-5-5" })114        let rows = [row("priced", input: 1, output: 2), row("unpriced")]115        var q = ModelQuery(); q.maxInputPrice = 100116        #expect(rows.applying(q).map(\.id) == ["priced"])117    }118119    @Test func contextFloorExcludesModelsWithoutAKnownContext() {120        let big = ids { $0.minContext = 1_000_000 }121        #expect(!big.isEmpty); #expect(big.contains("claude-opus-5-5")); #expect(!big.contains("or:liquid/lfm-3-1.7b")); #expect(!big.contains("gpt-5-pro"))122        #expect(ids { $0.minContext = 3_000_000 }.isEmpty)123        #expect(ids { $0.minContext = 2_000_000 } == ["or:openrouter/auto"])124    }125126    @Test func freeOpenWeightAndFavouriteSwitches() {127        #expect(Set(ids { $0.freeOnly = true }) == ["or:inclusionai/ling-3.2-flash:free", "or:baidu/ernie-5.5-flash-preview:free"])128        let open = ids { $0.openWeightOnly = true }129        #expect(open.count == 13); #expect(open.contains("or:mistralai/mistral-large-4-0")); #expect(!open.contains("claude-opus-5-5"))130        #expect(Set(ids { $0.favoritesOnly = true }) == ["claude-haiku-5-5", "or:google/gemini-4-neon"])131        #expect(ids { $0.freeOnly = true; $0.openWeightOnly = true; $0.capabilities = ["vision"] } == ["or:inclusionai/ling-3.2-flash:free"])132    }133134    @Test func filterBookkeeping() {135        var q = ModelQuery()136        #expect(!q.hasFilters); #expect(q.activeFilterCount == 0)137        q.text = "gemini"; q.sort = .context; q.includeNonChat = true138        #expect(!q.hasFilters)   // text, sort and « tous les types » are not filters139        q.makers = ["google", "meta"]; q.routes = ["openrouter"]; q.capabilities = ["vision"]; q.categories = ["coding"]; q.maxInputPrice = 3; q.minContext = 200_000; q.freeOnly = true; q.openWeightOnly = true; q.favoritesOnly = true140        #expect(q.hasFilters); #expect(q.activeFilterCount == 10)141        q.clearFilters()142        #expect(!q.hasFilters); #expect(q.activeFilterCount == 0); #expect(q.text == "gemini"); #expect(q.sort == .context); #expect(q.includeNonChat)143        #expect(q.makers.isEmpty && q.maxInputPrice == nil && q.minContext == nil && !q.freeOnly)144    }145146    // MARK: Sorting147148    @Test func sortByNameIsNaturalAndStable() {149        let rows = [row("b", name: "GPT 10"), row("a", name: "GPT 9"), row("d", name: "alpha"), row("c", name: "Alpha")]150        var q = ModelQuery(); q.sort = .name151        #expect(rows.applying(q).map(\.displayName).prefix(2).map { $0.lowercased() } == ["alpha", "alpha"])152        #expect(rows.applying(q).suffix(2).map(\.id) == ["a", "b"])   // 9 before 10153        let names = ids { $0.sort = .name }.compactMap { Self.catalog.model($0)?.displayName }154        #expect(names == names.sorted { $0.localizedStandardCompare($1) == .orderedAscending })155    }156157    @Test func sortByPricePutsUnknownPricesLastInBothDirections() {158        let rows = [row("mid", input: 3, output: 15), row("none"), row("cheap", input: 0.1, output: 0.4), row("half", input: 1), row("dear", input: 10, output: 50), row("free", input: 0, output: 0)]159        var q = ModelQuery(); q.sort = .priceAscending160        #expect(rows.applying(q).map(\.id) == ["free", "cheap", "mid", "dear", "half", "none"])161        q.sort = .priceDescending162        #expect(rows.applying(q).map(\.id) == ["dear", "mid", "cheap", "free", "half", "none"])163        let priced = all { $0.sort = .priceAscending }164        let known = priced.prefix { $0.blendedPrice != nil }165        #expect(priced.dropFirst(known.count).allSatisfy { $0.blendedPrice == nil }); #expect(known.count >= 25); #expect(known.allSatisfy { ($0.blendedPrice ?? 0) >= 0 }, "routers with a variable (negative sentinel) price are not priced")166        #expect(zip(known, known.dropFirst()).allSatisfy { ($0.blendedPrice ?? 0) <= ($1.blendedPrice ?? 0) })167    }168169    @Test func sortByContextPutsUnknownContextsLast() {170        let rows = [row("small", context: 8000), row("none"), row("big", context: 2_000_000), row("mid", context: 200_000)]171        var q = ModelQuery(); q.sort = .context172        #expect(rows.applying(q).map(\.id) == ["big", "mid", "small", "none"])173        let sorted = ids { $0.sort = .context }.compactMap { Self.catalog.model($0) }174        #expect(sorted.first?.id == "or:openrouter/auto"); #expect(sorted.last?.contextWindow == nil)175        let known = sorted.prefix { $0.contextWindow != nil }176        #expect(zip(known, known.dropFirst()).allSatisfy { ($0.contextWindow ?? 0) >= ($1.contextWindow ?? 0) }); #expect(sorted.dropFirst(known.count).allSatisfy { $0.contextWindow == nil })177    }178179    @Test func sortByNewestUsesThePublicationDateAndPutsUndatedModelsLast() {180        let rows = [row("or:v/old", created: 1_700_000_000), row("undated"), row("or:v/new", created: 1_790_000_000), row("or:v/mid", created: 1_750_000_000)]181        var q = ModelQuery(); q.sort = .newest182        #expect(rows.applying(q).map(\.id) == ["or:v/new", "or:v/mid", "or:v/old", "undated"])183        #expect(ids { $0.sort = .newest; $0.routes = ["openrouter"] }.first == "or:google/gemini-4-neon")184    }185186    @Test func sortByRecentUsePutsNeverUsedModelsLast() {187        let now = Date()188        let rows = [row("never"), row("yesterday", used: now.addingTimeInterval(-86_400)), row("justNow", used: now), row("lastWeek", used: now.addingTimeInterval(-7 * 86_400))]189        var q = ModelQuery(); q.sort = .recentlyUsed190        #expect(rows.applying(q).map(\.id) == ["justNow", "yesterday", "lastWeek", "never"])191        #expect(rows.recents(limit: 2).map(\.id) == ["justNow", "yesterday"])192    }193194    @Test func relevanceWithoutTextIsFavouritesThenVerifiedThenMakerAndName() {195        let rows = [row("or:zeta/z", name: "Zeta", status: "catalog"), row("b", name: "Beta"), row("or:acme/fav", name: "Catalogue favourite", favorite: true, status: "catalog"), row("a", name: "Alpha"), row("f", name: "Favourite", favorite: true)]196        #expect(rows.applying(ModelQuery()).map(\.id) == ["f", "or:acme/fav", "a", "b", "or:zeta/z"])197        let top = ids { _ in }198        #expect(top.first == "claude-haiku-5-5"); #expect(top[1] == "or:google/gemini-4-neon")   // the two favourites: verified first199        #expect(top[2..<6].allSatisfy { !$0.hasPrefix("or:") })200    }201202    // MARK: Search203204    @Test func searchRanksNamePrefixThenWordStartThenSubstringThenOtherFields() {205        let rows = [row("x-1", name: "Mega Flash"), row("x-2", name: "Flash 2"), row("x-3", name: "Superflash"), row("flash-backend", name: "Quiet"), row("x-5", name: "Other")]206        var q = ModelQuery(); q.text = "flash"207        #expect(rows.applying(q).map(\.id) == ["x-2", "x-1", "x-3", "flash-backend"])208        // an explicit sort overrides the ranking but keeps the matches209        q.sort = .name210        #expect(rows.applying(q).map(\.id) == ["x-2", "x-1", "flash-backend", "x-3"])211    }212213    @Test func searchIsInsensitiveToCaseAndDiacriticsAndNeedsEveryWord() {214        #expect(ids { $0.text = "GEMINI" } == ids { $0.text = "gemini" }); #expect(ids { $0.text = "gemini" }.count == 2)215        #expect(ids { $0.text = "gémini  néon" } == ["or:google/gemini-4-neon"])216        #expect(ids { $0.text = "gemini claude" }.isEmpty)217        #expect(ids { $0.text = "   " }.count == 29)218        // by maker name, by registry id, by catalogue slug219        #expect(Set(ids { $0.text = "moonshot" }) == ["or:moonshotai/kimi-k3.1-pro"])220        #expect(ids { $0.text = "deepseek-v4.1" } == ["or:deepseek/deepseek-v4.1-flash-1017"])221        #expect(ids { $0.text = "anthropic" }.count == 4)222        #expect(ids { $0.text = "embedding" }.isEmpty); #expect(all { $0.text = "embedding" }.map(\.id) == ["text-embedding-3-large"])223        // text and filters combine224        #expect(ids { $0.text = "flash"; $0.freeOnly = true }.count == 2); #expect(ids { $0.text = "flash"; $0.makers = ["tencent"] } == ["or:tencent/hy4-flash"])225    }226227    @Test func searchIndexCountsEveryRowAndMakersAreCountedMostFirst() {228        #expect(index.count == 34); #expect(ModelSearchIndex([]).search(ModelQuery()).isEmpty)229        let makers = models.makers230        #expect(makers.first?.slug == "openai"); #expect(makers.first?.count == 8); #expect(makers.first?.name == "OpenAI"); #expect(makers.first?.logo == "openai")231        #expect(makers.first { $0.slug == "anthropic" }?.count == 4); #expect(makers.first { $0.slug == "inclusionai" }?.logo == nil)232        #expect(makers.map(\.count).reduce(0, +) == 34); #expect(zip(makers, makers.dropFirst()).allSatisfy { $0.count >= $1.count })233        #expect(ModelSort.allCases.count == 7); #expect(Set(ModelSort.allCases.map(\.label)).count == 7)234    }235}236
Files main
Menu

Theme