SPB Git

Repository Public

ProviderTests.swift 16.5 KB 191 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/CoreTests/ProviderTests.swift3// Description: Providers and Live Model Intelligence: decoding of real payloads (providers, model detail by both routes), tolerance to unknown values, and the key flows (set, passkey re-auth replay, refusal, revoke with fallback) on a scripted transport.4import Foundation5import Testing6@testable import SPChatCore78@Suite struct ProviderTests {9    private func decode<T: Decodable>(_ type: T.Type, _ json: String) throws -> T { try JSONDecoder.api().decode(type, from: Data(json.utf8)) }1011    // MARK: Decoding — GET /api/providers1213    @Test func realProvidersPayload() throws {14        let o = try decode(ProvidersOverview.self, ExplorerFixtures.providers)15        #expect(o.providers.map(\.id) == ["openai", "anthropic", "xai", "openrouter", "tavily"])16        let openai = try #require(o.provider("openai"))17        #expect(openai.kind == "models"); #expect(openai.route == "direct"); #expect(openai.configured); #expect(openai.keySource == "env"); #expect(!openai.manageable)18        #expect(openai.models?.total == 144); #expect(openai.models?.chat == 30); #expect(openai.health?.state == "ok"); #expect(openai.validatedAt == nil)19        #expect(openai.keySourceLabel == "Clé de l'environnement du serveur")20        let or = try #require(o.provider("openrouter"))21        #expect(or.manageable); #expect(or.route == "openrouter"); #expect(or.keySource == "stored"); #expect(or.keyHint == "…T003"); #expect(or.validatedAt != nil)22        #expect(or.catalog?["total"]?.doubleValue == 444); #expect(or.catalog?["syncedAt"]?.stringValue.flatMap(ServerDate.parse) != nil); #expect(or.health == nil)23        let tavily = try #require(o.provider("tavily"))24        #expect(tavily.kind == "tool"); #expect(tavily.route == nil); #expect(tavily.models == nil); #expect(tavily.manageable)25        #expect(o.logos?.source == "@lobehub/icons-static-svg@1.104.0"); #expect(o.logos?.license?.hasPrefix("MIT") == true)26        #expect(o.logos?.makers.first { $0.slug == "google" }?.color == true); #expect(o.logos?.aliases["meta-llama"] == "meta")27        #expect(o.categoryRules.count == 12); #expect(o.categoryRules["long-context"] == "Fenêtre de contexte d'au moins 400 000 jetons.")28        #expect(Set(o.categoryRules.keys).isSubset(of: Set(ModelFacet.categories)), "every category the server explains has a label here (« local » came with the local route)")29    }3031    @Test func unknownProviderValuesAndMalformedRowsNeverBreakThePayload() throws {32        let json = #"{"providers":[{"id":"local","name":"MacLustr local","kind":"cluster","route":"lan","configured":"yes","keySource":"vault","manageable":true,"validatedAt":"not a date","models":{"total":"7"},"health":{"state":"warming-up","recentFailures":"2"},"future":{"a":1}},{"name":"no id"},42,{"id":"bare"}],"logos":{"makers":[{"slug":"x","name":"X","color":true},{"slug":7}]},"categoryRules":{"quantum":"Règle future."},"extra":true}"#33        let o = try decode(ProvidersOverview.self, json)34        #expect(o.providers.map(\.id) == ["local", "bare"])35        let p = o.providers[0]36        #expect(p.kind == "cluster"); #expect(p.route == "lan"); #expect(p.keySource == "vault"); #expect(p.keySourceLabel == "vault"); #expect(!p.configured); #expect(p.validatedAt == nil)37        #expect(p.models?.total == 7); #expect(p.models?.chat == 0); #expect(p.health?.state == "warming-up"); #expect(p.health?.recentFailures == 2)38        let bare = o.providers[1]39        #expect(bare.name == "bare"); #expect(bare.kind == "models"); #expect(bare.keySource == "none"); #expect(bare.keyHint == nil); #expect(!bare.manageable); #expect(bare.health == nil)40        #expect(o.logos?.makers.map(\.slug) == ["x"]); #expect(o.categoryRules["quantum"] == "Règle future.")41        // an empty object is still a valid overview42        let empty = try decode(ProvidersOverview.self, "{}")43        #expect(empty.providers.isEmpty); #expect(empty.logos == nil); #expect(empty.categoryRules.isEmpty)44    }4546    // MARK: Decoding — GET /api/models/detail4748    @Test func realDetailOfADirectModel() throws {49        let d = try decode(ModelDetail.self, ExplorerFixtures.detailDirect)50        #expect(d.model.id == "claude-haiku-5-5"); #expect(d.model.route == "direct"); #expect(d.model.isVerified); #expect(d.model.catalog == nil)51        #expect(d.model.benchmark?.passed == 11); #expect(d.model.benchmark?.perTest["vision"] == "pass"); #expect(d.model.benchmark?.p50TtftMs == 637)52        #expect(d.model.pricing?["cacheWrite1h"]?.doubleValue == 0.2); #expect(d.model.pricingNotes == "prompts >100K tokens: $0.50/$2.50"); #expect(d.model.cachedInputPrice == 0.01)53        #expect(d.params?.effort == ["low", "medium", "high", "xhigh", "max"]); #expect(d.params?.temperature == false); #expect(d.params?.reasoningSummary == true)54        #expect(d.params?.notes["temperature"]?.contains("Anthropic") == true)55        let m = try #require(d.measured)56        #expect(m.source == "Mesuré sur ce compte (journal d'utilisation de SP Chat)"); #expect(m.requests == 245); #expect(m.errors == 0); #expect(m.p50LatencyMs == 2423); #expect(m.p50TtftMs == 2304)57        #expect(m.inputTokens == 381_237); #expect(m.outputTokens == 151_261); #expect(abs((m.costUsd ?? 0) - 0.091678) < 1e-9); #expect(abs((m.tokensPerSec ?? 0) - 192.7278) < 1e-3)58        #expect(m.firstUsed != nil); #expect(m.lastUsed != nil)59        #expect(d.endpoints == nil); #expect(d.thirdPartyBenchmarks.isEmpty)60        let twin = try #require(d.twins.first)61        #expect(d.twins.count == 1); #expect(twin.id == "or:anthropic/claude-haiku-5.5"); #expect(twin.route == "openrouter"); #expect(twin.displayName == "Claude Haiku 5.5"); #expect(twin.status == "catalog")62        #expect(abs((twin.inputPrice ?? 0) - 0.1) < 1e-9); #expect(twin.outputPrice == 0.5); #expect(twin.pricing?["tiers"]?.arrayValue?.count == 1)63    }6465    @Test func realDetailOfAnOpenRouterModel() throws {66        let d = try decode(ModelDetail.self, ExplorerFixtures.detailOpenRouter)67        #expect(d.model.id == "or:google/gemini-4-neon"); #expect(d.model.isOpenRouter); #expect(d.model.isCatalogOnly); #expect(d.model.benchmark == nil); #expect(d.model.creatorName == "Google")68        let c = try #require(d.model.catalog)69        #expect(c.slug == "google/gemini-4-neon"); #expect(c.inputModalities.contains("video")); #expect(c.outputModalities == ["text"]); #expect(c.supportedParameters.contains("tool_choice"))70        #expect(c.tokenizer == "Gemini"); #expect(c.moderated == false); #expect(c.huggingFaceId == nil); #expect(c.aliasOf == nil); #expect(c.expirationDate == nil)71        #expect(c.reasoning?.mandatory == true); #expect(c.reasoning?.defaultEffort == "medium"); #expect(c.reasoning?.efforts == ["xhigh", "high", "medium", "low"])72        #expect(c.prices?["web_search"] == 0.014); #expect(c.prices?.count == 6); #expect(d.model.publishedAt != nil)73        #expect(d.params?.effortAlias == ["max": "xhigh"]); #expect(d.params?.temperature == true)74        #expect(d.measured?.requests == 5)75        let e = try #require(d.endpoints)76        #expect(e.source == "OpenRouter — points de terminaison en direct"); #expect(e.list.count == 5)77        #expect(e.list[0].provider == "Google AI Studio"); #expect(e.list[0].tag == "google-ai-studio/flex"); #expect(e.list[0].inputPerM == 1.5); #expect(e.list[0].outputPerM == 7.5)78        #expect(e.list[0].contextLength == 1_048_576); #expect(e.list[1].maxCompletionTokens == 65_536); #expect(e.list[1].uptimeLast30m == 90); #expect(e.list[1].status == -2)79        #expect(e.list[4].uptimeLast30m == nil); #expect(e.list[0].supportsTools == true); #expect(Set(e.list.map(\.id)).count == 5)80        let b = try #require(d.thirdPartyBenchmarks.first)81        #expect(d.thirdPartyBenchmarks.count == 1); #expect(b.source == "artificial_analysis"); #expect(b.sourceLabel == "Artificial Analysis"); #expect(b.values["intelligence_index"] == 61.4)82        #expect(b.figures.count == 7); #expect(b.figures.map(\.name) == b.figures.map(\.name).sorted())83        #expect(ModelDetail.BenchmarkSource.figureLabel("strategy_and_ops_index") == "Strategy and ops")84        #expect(d.twins.isEmpty)85    }8687    @Test func detailToleratesMissingBlocksAndMalformedRows() throws {88        let json = #"{"model":{"id":"or:new/thing","status":"quarantined","provider":"hyperscaler","catalog":{"benchmarks":{"available":["x"],"lab":{"score":"91.5","note":"n/a"},"empty":{}}}},"params":{"effort":"high"},"measured":{"requests":"3","p50_latency_ms":"1200.5"},"endpoints":{"list":[{"provider":"A","inputPerM":"0.5"},"junk",{"tag":"b/c"}]},"twins":[{"id":"thing"},{"route":"direct"},7]}"#89        let d = try decode(ModelDetail.self, json)90        #expect(d.model.status == "quarantined"); #expect(d.model.provider == "hyperscaler"); #expect(d.model.route == "openrouter"); #expect(d.model.creator == "new")91        #expect(d.params?.effort == []); #expect(d.measured?.requests == 3); #expect(d.measured?.p50LatencyMs == 1200.5); #expect(d.measured?.source == "")92        #expect(d.endpoints?.list.count == 2); #expect(d.endpoints?.list[0].inputPerM == 0.5); #expect(d.endpoints?.list[1].provider == "?"); #expect(d.endpoints?.source == "")93        #expect(d.twins.map(\.id) == ["thing"]); #expect(d.twins[0].route == "direct"); #expect(d.twins[0].displayName == "thing")94        // only sources that carry numbers are kept; the list form {"available": […]} is not a figure95        #expect(d.thirdPartyBenchmarks.map(\.source) == ["lab"]); #expect(d.thirdPartyBenchmarks[0].values == ["score": 91.5])96        let minimal = try decode(ModelDetail.self, #"{"model":{"id":"m"}}"#)97        #expect(minimal.params == nil); #expect(minimal.measured == nil); #expect(minimal.endpoints == nil); #expect(minimal.twins.isEmpty)98        #expect(throws: (any Error).self) { _ = try decode(ModelDetail.self, #"{"params":{}}"#) }99    }100101    @Test func detailRequestCarriesTheModelIdAsAQueryItem() async throws {102        let t = MockTransport(); let api = makeClient(t)103        t.on("GET", "/api/models/detail", .json(200, ExplorerFixtures.detailOpenRouter))104        let d = try await api.modelDetail("or:google/gemini-4-neon")105        #expect(d.model.id == "or:google/gemini-4-neon")106        let q = try #require(t.requests.first?.query)107        #expect(URLComponents(string: "http://x/?" + q)?.queryItems?.first { $0.name == "id" }?.value == "or:google/gemini-4-neon")108    }109110    // MARK: Key flows (never run against a real backend)111112    @Test func providersAndTestEndpoints() async throws {113        let t = MockTransport(); let api = makeClient(t)114        t.on("GET", "/api/providers", .json(200, ExplorerFixtures.providers))115        t.on("POST", "/api/providers/openrouter/test", .json(200, #"{"ok":true,"message":"Clé acceptée par OpenRouter (sans plafond de clé ; 0.0693 $ consommés).","info":{"limit":null,"usage":0.06930354,"isFreeTier":false,"expiresAt":null}}"#))116        t.on("POST", "/api/providers/tavily/test", .json(200, #"{"ok":false}"#))117        #expect(try await api.providers().providers.count == 5)118        let ok = try await api.testProvider("openrouter")119        #expect(ok.ok); #expect(ok.message.hasPrefix("Clé acceptée par OpenRouter")); #expect(ok.info?["usage"]?.doubleValue == 0.06930354); #expect(ok.info?["isFreeTier"]?.boolValue == false)120        let ko = try await api.testProvider("tavily")121        #expect(!ko.ok); #expect(ko.message.isEmpty); #expect(ko.info == nil)122        #expect(t.requests.map(\.method) == ["GET", "POST", "POST"]); #expect(t.requests[1].headers["Origin"] != nil)123    }124125    @Test func settingAKeySendsItOnceInTheBody() async throws {126        let t = MockTransport(); let api = makeClient(t)127        t.on("PUT", "/api/providers/openrouter/key", .json(200, #"{"ok":true,"provider":{"id":"openrouter","configured":true,"keySource":"stored","keyHint":"…9zzz"}}"#))128        try await api.setProviderKey("openrouter", key: "sk-or-test-0000")129        #expect(t.requests.count == 1)130        let r = try #require(t.requests.first)131        #expect(r.method == "PUT"); #expect(r.path == "/api/providers/openrouter/key"); #expect(r.query == nil); #expect(r.headers["Origin"] != nil)132        #expect(JSONValue.parse(r.body ?? Data()) == .object(["key": .string("sk-or-test-0000")]))133        // the key travels in the body only: never in the URL or a header134        #expect(!r.headers.values.contains { $0.contains("sk-or-test-0000") })135    }136137    @Test func reauthRequiredRunsThePasskeyConfirmationOnceThenReplaysTheKey() async throws {138        let t = MockTransport(); let api = makeClient(t); let count = Counter()139        api.reauthenticator = { count.increment() }140        t.on("PUT", "/api/providers/tavily/key", .json(403, #"{"error":"Confirme avec ta clé d'accès pour continuer","code":"reauth_required"}"#), .json(200, #"{"ok":true}"#))141        try await api.setProviderKey("tavily", key: "tvly-test-0000")142        #expect(count.value == 1); #expect(t.requests.count == 2)143        #expect(t.requests.allSatisfy { $0.method == "PUT" && JSONValue.parse($0.body ?? Data())?["key"]?.stringValue == "tvly-test-0000" })144    }145146    @Test func reauthIsNeverLoopedAndItsFailureStopsTheKeyChange() async {147        let t = MockTransport(); let api = makeClient(t); let count = Counter()148        api.reauthenticator = { count.increment() }149        t.on("PUT", "/api/providers/openrouter/key", .json(403, #"{"error":"Confirme","code":"reauth_required"}"#))150        do { try await api.setProviderKey("openrouter", key: "k"); Issue.record("must throw") } catch let e as APIError { #expect(e.kind == .reauthRequired) } catch { Issue.record("wrong error") }151        #expect(count.value == 1); #expect(t.requests.count == 2)152        // the owner cancels the passkey sheet: the request is not replayed153        let t2 = MockTransport(); let api2 = makeClient(t2)154        api2.reauthenticator = { throw APIError(.cancelled, message: "Annulé") }155        t2.on("PUT", "/api/providers/openrouter/key", .json(403, #"{"error":"Confirme","code":"reauth_required"}"#), .json(200, #"{"ok":true}"#))156        do { try await api2.setProviderKey("openrouter", key: "k"); Issue.record("must throw") } catch let e as APIError { #expect(e.kind == .cancelled) } catch { Issue.record("wrong error") }157        #expect(t2.requests.count == 1)158    }159160    @Test func aRefusedKeyThrowsTheProvidersMessage() async {161        let t = MockTransport(); let api = makeClient(t)162        t.on("PUT", "/api/providers/openrouter/key", .json(422, #"{"ok":false,"message":"OpenRouter a refusé cette clé (401 : No auth credentials found)."}"#))163        do { try await api.setProviderKey("openrouter", key: "bad"); Issue.record("must throw") }164        catch let e as APIError { #expect(e.status == 422); #expect(e.message == "OpenRouter a refusé cette clé (401 : No auth credentials found)."); #expect(e.code == "key_refused") }165        catch { Issue.record("wrong error") }166        #expect(t.requests.count == 1)167        // no message in the body: a clear default, never « HTTP 422 »168        let t2 = MockTransport(); let api2 = makeClient(t2)169        t2.on("PUT", "/api/providers/tavily/key", .json(422, "{}"))170        do { try await api2.setProviderKey("tavily", key: "bad"); Issue.record("must throw") }171        catch let e as APIError { #expect(e.status == 422); #expect(e.message == "Le fournisseur a refusé cette clé.") } catch { Issue.record("wrong error") }172        // any other failure keeps its own status and message173        let t3 = MockTransport(); let api3 = makeClient(t3)174        t3.on("PUT", "/api/providers/openai/key", .json(400, #"{"error":"Ce fournisseur n'est pas géré depuis les apps"}"#))175        do { try await api3.setProviderKey("openai", key: "k"); Issue.record("must throw") }176        catch let e as APIError { #expect(e.status == 400); #expect(e.message == "Ce fournisseur n'est pas géré depuis les apps") } catch { Issue.record("wrong error") }177    }178179    @Test func revokingAKeyReportsTheFallback() async throws {180        let t = MockTransport(); let api = makeClient(t); let count = Counter()181        api.reauthenticator = { count.increment() }182        t.on("DELETE", "/api/providers/openrouter/key", .json(403, #"{"error":"Confirme","code":"reauth_required"}"#), .json(200, #"{"removed":true,"fallback":true}"#))183        t.on("DELETE", "/api/providers/tavily/key", .json(200, #"{"removed":true,"fallback":false}"#))184        let a = try await api.removeProviderKey("openrouter")185        #expect(a.removed); #expect(a.fallback); #expect(count.value == 1)186        let b = try await api.removeProviderKey("tavily")187        #expect(b.removed); #expect(!b.fallback)188        #expect(t.requests.map(\.method) == ["DELETE", "DELETE", "DELETE"]); #expect(t.requests.allSatisfy { JSONValue.parse($0.body ?? Data("{}".utf8)) == .object([:]) })189    }190}191
Files main
Menu

Theme