Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/CoreTests/ModelQueryTests.swift3// Description: Catalogue query engine of the Explorer: every filter, every sort (unknown values last), search ranking, maker facet, and decoding that survives unknown values and malformed rows — on a real trimmed catalogue.4import Foundation5import Testing6@testable import SPChatCore78@Suite struct ModelQueryTests {9 private static let catalog: ModelCatalog = { (try? JSONDecoder.api().decode(ModelCatalog.self, from: Data(ExplorerFixtures.catalog.utf8))) ?? ModelCatalog() }()10 private var models: [AIModel] { Self.catalog.models }11 private var index: ModelSearchIndex { ModelSearchIndex(models) }12 private func ids(_ configure: (inout ModelQuery) -> Void) -> [String] { var q = ModelQuery(); configure(&q); return index.search(q).map(\.id) }13 private func all(_ configure: (inout ModelQuery) -> Void) -> [AIModel] { var q = ModelQuery(); q.includeNonChat = true; configure(&q); return index.search(q) }1415 /// Small hand-built rows for the orderings whose expected result must be obvious.16 private func row(_ id: String, name: String? = nil, input: Double? = nil, output: Double? = nil, context: Int? = nil, used: Date? = nil, created: Int? = nil, favorite: Bool = false, status: String = "verified") -> AIModel {17 var pricing: [String: JSONValue]?18 if input != nil || output != nil { pricing = [:]; if let input { pricing?["input"] = .number(input) }; if let output { pricing?["output"] = .number(output) } }19 return AIModel(id: id, provider: id.hasPrefix("or:") ? "openrouter" : "openai", displayName: name ?? id, status: status, contextWindow: context, pricing: pricing, favorite: favorite,20 catalog: created.map { CatalogInfo(slug: id, created: $0) }, lastUsedAt: used)21 }2223 // MARK: Decoding2425 @Test func realCatalogueDecodes() {26 #expect(models.count == 34); #expect(Self.catalog.defaultModel == "claude-opus-5-5"); #expect(Self.catalog.dock == ["claude-haiku-5-5", "or:google/gemini-4-neon"])27 #expect(Self.catalog.openrouter?.configured == true); #expect(Self.catalog.openrouter?.catalog?.total == 444); #expect(Self.catalog.refreshedAt != nil)28 #expect(models.filter(\.isOpenRouter).count == 24); #expect(models.filter(\.isVerified).count == 10); #expect(models.filter(\.isCatalogOnly).count == 24)29 let neon = Self.catalog.model("or:google/gemini-4-neon")30 #expect(neon?.creator == "google"); #expect(neon?.logo == "google"); #expect(neon?.route == "openrouter"); #expect(neon?.catalog?.benchmarkSources == ["artificial_analysis"] || neon?.catalog?.benchmarkSources.isEmpty == false)31 #expect(Self.catalog.model("or:inclusionai/ling-3.2-flash:free")?.logo == nil); #expect(Self.catalog.model("or:inclusionai/ling-3.2-flash:free")?.isFree == true)32 // the cache encoding gives the same catalogue back (dates are compared apart: they are re-encoded to the millisecond)33 func undated(_ m: AIModel) -> AIModel { var c = m; c.lastVerifiedAt = nil; c.lastUsedAt = nil; return c }34 let back = try? JSONDecoder.api().decode(ModelCatalog.self, from: JSONEncoder.api().encode(Self.catalog))35 #expect(back?.models.map(undated) == models.map(undated)); #expect(back?.dock == Self.catalog.dock); #expect(back?.defaultModel == Self.catalog.defaultModel)36 #expect(zip(back?.models ?? [], models).allSatisfy { abs(($0.lastVerifiedAt?.timeIntervalSince1970 ?? 0) - ($1.lastVerifiedAt?.timeIntervalSince1970 ?? 0)) < 0.002 })37 }3839 @Test func unknownValuesAndMalformedRowsNeverBreakTheCatalogue() throws {40 let json = #"{"models":[{"id":"ok-1","provider":"openai","displayName":"OK","kind":"chat","status":"verified","available":true},{"displayName":"no id"},"junk",null,{"id":"future-1","provider":"hyperscaler","kind":"hologram","status":"quarantined","available":true,"route":"satellite","contextWindow":"200000","pricing":{"input":"1.5","output":null,"tiers":"n/a"},"capabilities":{"vision":"unknown","tools":true},"categories":["quantum"],"catalog":{"created":"1790000000","prices":{"image":"0.01","bad":{}}},"benchmark":"none","lastUsedAt":"yesterday"},{"id":"or:new-vendor/model-x"}],"dock":"none","openrouter":{"configured":"maybe"}}"#41 let c = try JSONDecoder.api().decode(ModelCatalog.self, from: Data(json.utf8))42 #expect(c.models.map(\.id) == ["ok-1", "future-1", "or:new-vendor/model-x"]); #expect(c.dock.isEmpty); #expect(c.openrouter?.configured == false)43 let f = c.models[1]44 #expect(f.provider == "hyperscaler"); #expect(f.status == "quarantined"); #expect(f.kind == "hologram"); #expect(f.route == "satellite"); #expect(f.contextWindow == 200_000)45 #expect(f.inputPrice == 1.5); #expect(f.outputPrice == nil); #expect(f.blendedPrice == nil); #expect(!f.has("vision")); #expect(f.has("tools")); #expect(f.categories == ["quantum"])46 #expect(f.catalog?.created == 1_790_000_000); #expect(f.catalog?.prices == ["image": 0.01]); #expect(f.benchmark == nil); #expect(f.lastUsedAt == nil)47 #expect(!f.isVerified); #expect(!f.isCatalogOnly); #expect(!f.isSelectableForChat)48 let bare = c.models[2]49 #expect(bare.route == "openrouter"); #expect(bare.creator == "new-vendor"); #expect(bare.displayName == "or:new-vendor/model-x"); #expect(bare.status == "untested"); #expect(!bare.available)50 // unknown values are searchable and filterable like any other51 var q = ModelQuery(); q.includeNonChat = true; q.routes = ["satellite"]52 #expect(c.models.applying(q).map(\.id) == ["future-1"])53 q = ModelQuery(); q.includeNonChat = true; q.categories = ["quantum"]; q.text = "hyperscaler"54 #expect(c.models.applying(q).map(\.id) == ["future-1"])55 #expect(ModelFacet.label("quantum") == "Quantum"); #expect(ModelFacet.symbol("quantum") == "tag"); #expect(ModelFacet.capabilityLabel("telepathy") == "telepathy")56 }5758 // MARK: Filters5960 @Test func defaultQueryKeepsOnlyModelsThatCanBePickedForAConversation() {61 let picked = ids { _ in }62 #expect(picked.count == 29) // 34 rows − image, tts, stt, embedding, moderation63 #expect(!picked.contains("chatgpt-image-latest")); #expect(!picked.contains("text-embedding-3-large")); #expect(picked.contains("gpt-5-pro")); #expect(picked.contains("gpt-5-search-api"))64 #expect(all { _ in }.count == 34)65 // hidden and unavailable rows only appear with « tous les types »66 var rows = [row("a"), row("b"), row("c")]; rows[1].hidden = true; rows[2].available = false67 #expect(rows.applying(ModelQuery()).map(\.id) == ["a"])68 var q = ModelQuery(); q.includeNonChat = true69 #expect(Set(rows.applying(q).map(\.id)) == ["a", "b", "c"])70 }7172 @Test func makerFilter() {73 let google = ids { $0.makers = ["google"] }74 #expect(Set(google) == ["or:google/gemini-4-neon", "or:google/gemini-4-carbon"])75 let two = ids { $0.makers = ["anthropic", "meta"] }76 #expect(Set(two) == ["claude-haiku-5-5", "claude-opus-5-5", "claude-fable-5", "or:anthropic/claude-haiku-5.5", "or:meta/muse-ember"])77 #expect(ids { $0.makers = ["nobody"] }.isEmpty)78 }7980 @Test func routeFilter() {81 let direct = ids { $0.routes = ["direct"] }, or = ids { $0.routes = ["openrouter"] }82 #expect(direct.count == 5); #expect(direct.allSatisfy { !$0.hasPrefix("or:") })83 #expect(or.count == 24); #expect(or.allSatisfy { $0.hasPrefix("or:") })84 #expect(ids { $0.routes = ["direct", "openrouter"] }.count == 29)85 // the same model by two routes stays two rows86 let haiku = ids { $0.text = "claude haiku 5" }87 #expect(Set(haiku) == ["claude-haiku-5-5", "or:anthropic/claude-haiku-5.5"])88 }8990 @Test func capabilityFilterRequiresEveryFlag() {91 let vision = all { $0.capabilities = ["vision"] }92 #expect(!vision.isEmpty); #expect(vision.allSatisfy { $0.has("vision") })93 // « unknown » is not true94 #expect(!vision.contains { $0.id == "gpt-5-pro" })95 let both = all { $0.capabilities = ["vision", "audioInput"] }96 #expect(!both.isEmpty); #expect(both.count < vision.count); #expect(both.allSatisfy { $0.has("vision") && $0.has("audioInput") })97 #expect(all { $0.capabilities = ["imageGeneration"] }.map(\.id).contains("chatgpt-image-latest"))98 #expect(ids { $0.capabilities = ["telepathy"] }.isEmpty)99 }100101 @Test func categoryFilterRequiresEveryCategory() {102 let coding = ids { $0.categories = ["coding"] }103 #expect(coding.count == 7); #expect(coding.contains("or:tencent/hy4-flash"))104 let both = ids { $0.categories = ["coding", "research"] }105 #expect(Set(both) == ["or:tencent/hy4-flash", "or:moonshotai/kimi-k3.1-pro", "or:deepseek/deepseek-v4.1-flash-1017"])106 #expect(ids { $0.categories = ["fast"] }.allSatisfy { !$0.hasPrefix("or:") })107 }108109 @Test func priceCeilingExcludesModelsWithoutAPrice() {110 let cheap = all { $0.maxInputPrice = 0.1 }111 #expect(!cheap.isEmpty); #expect(cheap.allSatisfy { ($0.inputPrice ?? .infinity) <= 0.1 })112 #expect(!cheap.contains { $0.id == "gpt-4o-mini-transcribe" }) // no input price published113 #expect(cheap.contains { $0.id == "or:deepseek/deepseek-v4.1-flash-1017" }); #expect(!cheap.contains { $0.id == "claude-opus-5-5" })114 let rows = [row("priced", input: 1, output: 2), row("unpriced")]115 var q = ModelQuery(); q.maxInputPrice = 100116 #expect(rows.applying(q).map(\.id) == ["priced"])117 }118119 @Test func contextFloorExcludesModelsWithoutAKnownContext() {120 let big = ids { $0.minContext = 1_000_000 }121 #expect(!big.isEmpty); #expect(big.contains("claude-opus-5-5")); #expect(!big.contains("or:liquid/lfm-3-1.7b")); #expect(!big.contains("gpt-5-pro"))122 #expect(ids { $0.minContext = 3_000_000 }.isEmpty)123 #expect(ids { $0.minContext = 2_000_000 } == ["or:openrouter/auto"])124 }125126 @Test func freeOpenWeightAndFavouriteSwitches() {127 #expect(Set(ids { $0.freeOnly = true }) == ["or:inclusionai/ling-3.2-flash:free", "or:baidu/ernie-5.5-flash-preview:free"])128 let open = ids { $0.openWeightOnly = true }129 #expect(open.count == 13); #expect(open.contains("or:mistralai/mistral-large-4-0")); #expect(!open.contains("claude-opus-5-5"))130 #expect(Set(ids { $0.favoritesOnly = true }) == ["claude-haiku-5-5", "or:google/gemini-4-neon"])131 #expect(ids { $0.freeOnly = true; $0.openWeightOnly = true; $0.capabilities = ["vision"] } == ["or:inclusionai/ling-3.2-flash:free"])132 }133134 @Test func filterBookkeeping() {135 var q = ModelQuery()136 #expect(!q.hasFilters); #expect(q.activeFilterCount == 0)137 q.text = "gemini"; q.sort = .context; q.includeNonChat = true138 #expect(!q.hasFilters) // text, sort and « tous les types » are not filters139 q.makers = ["google", "meta"]; q.routes = ["openrouter"]; q.capabilities = ["vision"]; q.categories = ["coding"]; q.maxInputPrice = 3; q.minContext = 200_000; q.freeOnly = true; q.openWeightOnly = true; q.favoritesOnly = true140 #expect(q.hasFilters); #expect(q.activeFilterCount == 10)141 q.clearFilters()142 #expect(!q.hasFilters); #expect(q.activeFilterCount == 0); #expect(q.text == "gemini"); #expect(q.sort == .context); #expect(q.includeNonChat)143 #expect(q.makers.isEmpty && q.maxInputPrice == nil && q.minContext == nil && !q.freeOnly)144 }145146 // MARK: Sorting147148 @Test func sortByNameIsNaturalAndStable() {149 let rows = [row("b", name: "GPT 10"), row("a", name: "GPT 9"), row("d", name: "alpha"), row("c", name: "Alpha")]150 var q = ModelQuery(); q.sort = .name151 #expect(rows.applying(q).map(\.displayName).prefix(2).map { $0.lowercased() } == ["alpha", "alpha"])152 #expect(rows.applying(q).suffix(2).map(\.id) == ["a", "b"]) // 9 before 10153 let names = ids { $0.sort = .name }.compactMap { Self.catalog.model($0)?.displayName }154 #expect(names == names.sorted { $0.localizedStandardCompare($1) == .orderedAscending })155 }156157 @Test func sortByPricePutsUnknownPricesLastInBothDirections() {158 let rows = [row("mid", input: 3, output: 15), row("none"), row("cheap", input: 0.1, output: 0.4), row("half", input: 1), row("dear", input: 10, output: 50), row("free", input: 0, output: 0)]159 var q = ModelQuery(); q.sort = .priceAscending160 #expect(rows.applying(q).map(\.id) == ["free", "cheap", "mid", "dear", "half", "none"])161 q.sort = .priceDescending162 #expect(rows.applying(q).map(\.id) == ["dear", "mid", "cheap", "free", "half", "none"])163 let priced = all { $0.sort = .priceAscending }164 let known = priced.prefix { $0.blendedPrice != nil }165 #expect(priced.dropFirst(known.count).allSatisfy { $0.blendedPrice == nil }); #expect(known.count >= 25); #expect(known.allSatisfy { ($0.blendedPrice ?? 0) >= 0 }, "routers with a variable (negative sentinel) price are not priced")166 #expect(zip(known, known.dropFirst()).allSatisfy { ($0.blendedPrice ?? 0) <= ($1.blendedPrice ?? 0) })167 }168169 @Test func sortByContextPutsUnknownContextsLast() {170 let rows = [row("small", context: 8000), row("none"), row("big", context: 2_000_000), row("mid", context: 200_000)]171 var q = ModelQuery(); q.sort = .context172 #expect(rows.applying(q).map(\.id) == ["big", "mid", "small", "none"])173 let sorted = ids { $0.sort = .context }.compactMap { Self.catalog.model($0) }174 #expect(sorted.first?.id == "or:openrouter/auto"); #expect(sorted.last?.contextWindow == nil)175 let known = sorted.prefix { $0.contextWindow != nil }176 #expect(zip(known, known.dropFirst()).allSatisfy { ($0.contextWindow ?? 0) >= ($1.contextWindow ?? 0) }); #expect(sorted.dropFirst(known.count).allSatisfy { $0.contextWindow == nil })177 }178179 @Test func sortByNewestUsesThePublicationDateAndPutsUndatedModelsLast() {180 let rows = [row("or:v/old", created: 1_700_000_000), row("undated"), row("or:v/new", created: 1_790_000_000), row("or:v/mid", created: 1_750_000_000)]181 var q = ModelQuery(); q.sort = .newest182 #expect(rows.applying(q).map(\.id) == ["or:v/new", "or:v/mid", "or:v/old", "undated"])183 #expect(ids { $0.sort = .newest; $0.routes = ["openrouter"] }.first == "or:google/gemini-4-neon")184 }185186 @Test func sortByRecentUsePutsNeverUsedModelsLast() {187 let now = Date()188 let rows = [row("never"), row("yesterday", used: now.addingTimeInterval(-86_400)), row("justNow", used: now), row("lastWeek", used: now.addingTimeInterval(-7 * 86_400))]189 var q = ModelQuery(); q.sort = .recentlyUsed190 #expect(rows.applying(q).map(\.id) == ["justNow", "yesterday", "lastWeek", "never"])191 #expect(rows.recents(limit: 2).map(\.id) == ["justNow", "yesterday"])192 }193194 @Test func relevanceWithoutTextIsFavouritesThenVerifiedThenMakerAndName() {195 let rows = [row("or:zeta/z", name: "Zeta", status: "catalog"), row("b", name: "Beta"), row("or:acme/fav", name: "Catalogue favourite", favorite: true, status: "catalog"), row("a", name: "Alpha"), row("f", name: "Favourite", favorite: true)]196 #expect(rows.applying(ModelQuery()).map(\.id) == ["f", "or:acme/fav", "a", "b", "or:zeta/z"])197 let top = ids { _ in }198 #expect(top.first == "claude-haiku-5-5"); #expect(top[1] == "or:google/gemini-4-neon") // the two favourites: verified first199 #expect(top[2..<6].allSatisfy { !$0.hasPrefix("or:") })200 }201202 // MARK: Search203204 @Test func searchRanksNamePrefixThenWordStartThenSubstringThenOtherFields() {205 let rows = [row("x-1", name: "Mega Flash"), row("x-2", name: "Flash 2"), row("x-3", name: "Superflash"), row("flash-backend", name: "Quiet"), row("x-5", name: "Other")]206 var q = ModelQuery(); q.text = "flash"207 #expect(rows.applying(q).map(\.id) == ["x-2", "x-1", "x-3", "flash-backend"])208 // an explicit sort overrides the ranking but keeps the matches209 q.sort = .name210 #expect(rows.applying(q).map(\.id) == ["x-2", "x-1", "flash-backend", "x-3"])211 }212213 @Test func searchIsInsensitiveToCaseAndDiacriticsAndNeedsEveryWord() {214 #expect(ids { $0.text = "GEMINI" } == ids { $0.text = "gemini" }); #expect(ids { $0.text = "gemini" }.count == 2)215 #expect(ids { $0.text = "gémini néon" } == ["or:google/gemini-4-neon"])216 #expect(ids { $0.text = "gemini claude" }.isEmpty)217 #expect(ids { $0.text = " " }.count == 29)218 // by maker name, by registry id, by catalogue slug219 #expect(Set(ids { $0.text = "moonshot" }) == ["or:moonshotai/kimi-k3.1-pro"])220 #expect(ids { $0.text = "deepseek-v4.1" } == ["or:deepseek/deepseek-v4.1-flash-1017"])221 #expect(ids { $0.text = "anthropic" }.count == 4)222 #expect(ids { $0.text = "embedding" }.isEmpty); #expect(all { $0.text = "embedding" }.map(\.id) == ["text-embedding-3-large"])223 // text and filters combine224 #expect(ids { $0.text = "flash"; $0.freeOnly = true }.count == 2); #expect(ids { $0.text = "flash"; $0.makers = ["tencent"] } == ["or:tencent/hy4-flash"])225 }226227 @Test func searchIndexCountsEveryRowAndMakersAreCountedMostFirst() {228 #expect(index.count == 34); #expect(ModelSearchIndex([]).search(ModelQuery()).isEmpty)229 let makers = models.makers230 #expect(makers.first?.slug == "openai"); #expect(makers.first?.count == 8); #expect(makers.first?.name == "OpenAI"); #expect(makers.first?.logo == "openai")231 #expect(makers.first { $0.slug == "anthropic" }?.count == 4); #expect(makers.first { $0.slug == "inclusionai" }?.logo == nil)232 #expect(makers.map(\.count).reduce(0, +) == 34); #expect(zip(makers, makers.dropFirst()).allSatisfy { $0.count >= $1.count })233 #expect(ModelSort.allCases.count == 7); #expect(Set(ModelSort.allCases.map(\.label)).count == 7)234 }235}236