Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/LocalChatDataTests/ModelHubAPITests.swift3// Description: Hub client and metadata rules: decoding of real response shapes, search URLs, error mapping, name parsing, candidates.4import Foundation5import LocalChatCore6import Testing7@testable import LocalChatModelHub89// Trimmed copies of real answers of https://huggingface.co/api/models/<repo>?blobs=true (captured 2026-11-20).10private let mlxDetailJSON = #"""11{"_id":"x","id":"mlx-community/Qwen3-4B-4bit","private":false,"pipeline_tag":"text-generation","library_name":"mlx",12 "tags":["mlx","safetensors","qwen3","text-generation","conversational","base_model:Qwen/Qwen3-4B","base_model:quantized:Qwen/Qwen3-4B","license:apache-2.0","4-bit","region:us"],13 "downloads":28747,"likes":17,"modelId":"mlx-community/Qwen3-4B-4bit","author":"mlx-community","sha":"4dcb3d101c2a062e5c1d4bb173588c54ea6c4d25",14 "lastModified":"2025-04-28T21:31:30.000Z","gated":false,"disabled":false,15 "config":{"architectures":["Qwen3ForCausalLM"],"model_type":"qwen3","quantization_config":{"bits":4},16 "tokenizer_config":{"bos_token":null,"eos_token":"<|im_end|>","chat_template":"{%- if tools %}…{%- if enable_thinking is defined and enable_thinking is false %}<think>"}},17 "cardData":{"library_name":"mlx","license":"apache-2.0","pipeline_tag":"text-generation","base_model":"Qwen/Qwen3-4B","tags":["mlx"]},18 "siblings":[{"rfilename":".gitattributes","blobId":"a","size":1570},{"rfilename":"README.md","blobId":"b","size":857},19 {"rfilename":"added_tokens.json","blobId":"c","size":707},{"rfilename":"config.json","blobId":"d","size":937},20 {"rfilename":"merges.txt","blobId":"e","size":1671853},21 {"rfilename":"model.safetensors","blobId":"f","size":2263022529,"lfs":{"sha256":"e240c0bdc0ebb0681bf0da0f98d9719fd6ebe269a3633f81542c13e81345651d","size":2263022529,"pointerSize":135}},22 {"rfilename":"model.safetensors.index.json","blobId":"g","size":63924},{"rfilename":"special_tokens_map.json","blobId":"h","size":613},23 {"rfilename":"tokenizer.json","blobId":"i","size":11422654,"lfs":{"sha256":"aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4","size":11422654,"pointerSize":133}},24 {"rfilename":"tokenizer_config.json","blobId":"j","size":9706},{"rfilename":"vocab.json","blobId":"k","size":2776833}],25 "createdAt":"2025-04-28T21:28:51.000Z","safetensors":{"parameters":{"U32":4022272000,"BF16":196096},"total":4022468096},"usedStorage":2274445183}26"""#2728private let ggufDetailJSON = #"""29{"id":"unsloth/Qwen3-4B-GGUF","private":false,"pipeline_tag":"text-generation","library_name":"transformers","tags":["transformers","gguf","qwen3","license:apache-2.0"],30 "downloads":100,"likes":5,"author":"unsloth","sha":"22c9fc8a8c7700b76a1789366280a6a5a1ad1120","lastModified":"2025-05-01T10:00:00.000Z","gated":false,31 "config":{"architectures":"['Qwen3ForCausalLM']","model_type":"qwen3"},32 "cardData":{"base_model":"Qwen/Qwen3-4B","license":"apache-2.0"},33 "gguf":{"total":"4022468096","architecture":"qwen3","context_length":"40960","chat_template":"{%- if tools %}x{%- endif %}","eos_token":"<|im_end|>"},34 "siblings":[{"rfilename":"Qwen3-4B-Q4_K_M.gguf","size":2497281312,"lfs":{"sha256":"f6f851777709aa","size":2497281312}},35 {"rfilename":"Qwen3-4B-Q8_0.gguf","size":4280405792,"lfs":{"sha256":"eed555233267bb","size":4280405792}},36 {"rfilename":"Qwen3-4B-UD-Q4_K_XL.gguf","size":2546341152,"lfs":{"sha256":"f6e3fb6c2cdc","size":2546341152}},37 {"rfilename":"big-Q6_K-00001-of-00002.gguf","size":100,"lfs":{"sha256":"aa","size":100}},38 {"rfilename":"big-Q6_K-00002-of-00002.gguf","size":50,"lfs":{"sha256":"bb","size":50}},39 {"rfilename":"orphan-Q5_K_M-00002-of-00003.gguf","size":50,"lfs":{"sha256":"cc","size":50}},40 {"rfilename":"README.md","size":19797},{"rfilename":"imatrix_unsloth.dat","size":3842218,"lfs":{"sha256":"e21e","size":3842218}}]}41"""#4243@Suite struct ModelHubAPITests {44 @Test func decodesMLXRepositoryDetail() throws {45 let model = try JSONDecoder().decode(HubModel.self, from: Data(mlxDetailJSON.utf8))46 #expect(model.id == "mlx-community/Qwen3-4B-4bit")47 #expect(model.commit == "4dcb3d101c2a062e5c1d4bb173588c54ea6c4d25")48 #expect(model.gating == .open && !model.isGated)49 #expect(model.license == "apache-2.0")50 #expect(model.modelType == "qwen3")51 #expect(model.safetensorsParameters == 4_022_468_096)52 #expect(model.files.count == 11)53 let weights = try #require(model.files.first { $0.path == "model.safetensors" })54 #expect(weights.size == 2_263_022_529 && weights.sha256?.hasPrefix("e240c0bd") == true && weights.isLFS)55 #expect(model.files.first { $0.path == "config.json" }?.sha256 == nil)56 #expect(model.lastModified != nil)57 #expect(model.baseModel == "Qwen/Qwen3-4B")58 #expect(model.chatTemplate?.contains("enable_thinking") == true)59 }6061 @Test func decodesGatingVariants() throws {62 func gating(_ json: String) throws -> HubModel.Gating { try JSONDecoder().decode(HubModel.self, from: Data(json.utf8)).gating }63 #expect(try gating(#"{"id":"a/b","gated":"manual"}"#) == .manual)64 #expect(try gating(#"{"id":"a/b","gated":"auto"}"#) == .auto)65 #expect(try gating(#"{"id":"a/b","gated":false}"#) == .open)66 #expect(try gating(#"{"id":"a/b"}"#) == .open)67 }6869 @Test func buildsSearchURL() {70 let api = HuggingFaceAPI(session: HubStub.session())71 let url = api.searchURL(HubSearchQuery(text: " qwen3 ", format: .gguf, pipelineTag: "text-generation", sort: .likes, limit: 500))72 let items = Dictionary(uniqueKeysWithValues: URLComponents(url: url, resolvingAgainstBaseURL: false)!.queryItems!.map { ($0.name, $0.value ?? "") })73 #expect(url.path == "/api/models")74 #expect(items["search"] == "qwen3" && items["filter"] == "gguf" && items["pipeline_tag"] == "text-generation")75 #expect(items["sort"] == "likes" && items["limit"] == "100" && items["full"] == "true" && items["config"] == "true")76 #expect(HuggingFaceAPI.nextLink(#"<https://huggingface.co/api/models?cursor=abc>; rel="next""#)?.query == "cursor=abc")77 #expect(HuggingFaceAPI.nextLink(nil) == nil)78 }7980 @Test func validatesRepositoryIDs() {81 #expect(HuggingFaceAPI.isValidRepositoryID("mlx-community/Qwen3-4B-4bit"))82 for bad in ["nope", "a/b/c", "../etc", "a/..", "a b/c", "/x", "a/"] { #expect(!HuggingFaceAPI.isValidRepositoryID(bad)) }83 }8485 @Test func mapsHubStatusCodes() {86 #expect(HuggingFaceAPI.error(status: 200, repo: "r", hasToken: false, errorCode: nil) == nil)87 #expect(HuggingFaceAPI.error(status: 401, repo: "r", hasToken: false, errorCode: "GatedRepo") == .gated("r"))88 #expect(HuggingFaceAPI.error(status: 403, repo: "r", hasToken: true, errorCode: nil) == .gated("r"))89 #expect(HuggingFaceAPI.error(status: 401, repo: "r", hasToken: false, errorCode: nil) == .notFound("r"))90 #expect(HuggingFaceAPI.error(status: 401, repo: "r", hasToken: true, errorCode: nil) == .unauthorized("r"))91 #expect(HuggingFaceAPI.error(status: 429, repo: "r", hasToken: false, errorCode: nil) == .rateLimited)92 #expect(HuggingFaceAPI.error(status: 503, repo: "r", hasToken: false, errorCode: nil)?.isTransient == true)93 #expect(HubError.gated("r").isTransient == false)94 }9596 @Test func searchesAndFetchesThroughTheSession() async throws {97 let base = HubStub.register { request in98 let path = request.url!.path99 if path == "/api/models" {100 // one malformed entry must not break the page101 let body = "[\(mlxDetailJSON),{\"nope\":1}]"102 return HubStubResponse(headers: ["Link": "<https://stub.test/api/models?cursor=n2>; rel=\"next\""], body: Data(body.utf8))103 }104 if path == "/api/models/acme/gated" { return HubStubResponse(status: 401, headers: ["X-Error-Code": "GatedRepo"]) }105 if path == "/api/models/acme/missing" { return HubStubResponse(status: 401) }106 if path.hasSuffix("/README.md") { return HubStubResponse(body: Data("---\nlicense: mit\n---\n# Title\nBody".utf8)) }107 return HubStubResponse(body: Data(mlxDetailJSON.utf8))108 }109 let api = HuggingFaceAPI(baseURL: base, session: HubStub.session(), tokenProvider: { "hf_test" })110 let page = try await api.search(HubSearchQuery(text: "qwen3"))111 #expect(page.models.count == 1 && page.next?.query == "cursor=n2")112 let detail = try await api.model("mlx-community/Qwen3-4B-4bit")113 #expect(detail.files.count == 11)114 #expect(HubStub.requests(base).allSatisfy { $0.value(forHTTPHeaderField: "Authorization") == "Bearer hf_test" })115 #expect(HubStub.requests(base).contains { $0.url?.query?.contains("blobs=true") == true })116 await #expect(throws: HubError.gated("acme/gated")) { try await api.model("acme/gated") }117 await #expect(throws: HubError.unauthorized("acme/missing")) { try await api.model("acme/missing") }118 await #expect(throws: HubError.invalidRepository("bad id")) { try await api.model("bad id") }119 #expect(try await api.readme("acme/any") == "# Title\nBody")120 }121}122123@Suite struct ModelHubNamingTests {124 @Test(arguments: [125 ("Qwen3-4B-Q4_K_M.gguf", "Q4_K_M", 4.85), ("model.Q8_0.gguf", "Q8_0", 8.5), ("Qwen3-4B-IQ4_XS.gguf", "IQ4_XS", 4.25),126 ("Qwen3-4B-UD-Q4_K_XL.gguf", "Q4_K_XL", 4.85), ("x-BF16.gguf", "BF16", 16), ("x-q2_k.gguf", "Q2_K", 2.96),127 ("llama-f16.gguf", "F16", 16), ("x-UD-IQ1_M.gguf", "IQ1_M", 1.75), ("x-Q3_K_L.gguf", "Q3_K_L", 4.27),128 ])129 func parsesGGUFQuantization(name: String, label: String, bits: Double) {130 let q = ModelNaming.ggufQuantization(fromName: name)131 #expect(q?.label == label)132 #expect(q?.bitsPerWeight == bits)133 }134135 @Test func ggufQuantizationRejectsLookalikes() {136 #expect(ModelNaming.ggufQuantization(fromName: "Qwen3-4B.gguf") == nil)137 #expect(ModelNaming.ggufQuantization(fromName: "mmproj-model.gguf") == nil)138 #expect(ModelNaming.ggufQuantization(fileType: 15)?.label == "Q4_K_M")139 #expect(ModelNaming.ggufQuantization(fileType: 7)?.label == "Q8_0")140 #expect(ModelNaming.ggufQuantization(fileType: 999) == nil)141 }142143 @Test(arguments: [144 ("Qwen3-4B-4bit", nil as Int?, "4bit", 4.5), ("gemma-3-1b-it-qat-4bit", nil, "4bit QAT", 4.5), ("Qwen3-Embedding-0.6B-4bit-DWQ", nil, "4bit DWQ", 4.5),145 ("Llama-3.2-3B-Instruct-bf16", nil, "bf16", 16), ("Model-8bit", nil, "8bit", 8.5), ("NoHint", 6, "6bit", 6.5), ("Model-MLX-8-bit", nil, "8bit", 8.5),146 ])147 func parsesMLXQuantization(name: String, bits: Int?, label: String, bpw: Double) {148 let q = ModelNaming.mlxQuantization(fromName: name, configBits: bits)149 #expect(q?.label == label)150 #expect(q?.bitsPerWeight == bpw)151 }152153 @Test func mlxQuantizationUnknownIsNil() {154 #expect(ModelNaming.mlxQuantization(fromName: "all-MiniLM-L6-v2") == nil)155 #expect(ModelNaming.mlxQuantization(fromName: "habit-tracker") == nil)156 }157158 @Test(arguments: [159 ("Qwen3-4B-4bit", 4_000_000_000 as Int64?), ("Qwen2.5-1.5B-Instruct", 1_500_000_000), ("SmolVLM2-500M-Video-Instruct-mlx", 500_000_000),160 ("Qwen3-Coder-30B-A3B-Instruct-4bit", 30_000_000_000), ("Qwen3-0.6b", 600_000_000), ("gemma-3n-E2B-it-4bit", nil),161 ("all-MiniLM-L6-v2", nil), ("bge-small-en-v1.5", nil), ("Llama-3.2-3B-Instruct-Q4_K_M.gguf", 3_000_000_000), ("model-4bit", nil), ("LFM2-350M", 350_000_000),162 ])163 func parsesParameterCounts(name: String, expected: Int64?) {164 #expect(ModelNaming.parameterCount(fromName: name) == expected)165 }166167 @Test func capabilityRules() {168 func caps(_ name: String, pipeline: String? = nil, template: String? = nil, vision: Bool = false, projector: Bool = false) -> Set<ModelCapability> {169 ModelNaming.capabilities(name: name, pipelineTag: pipeline, tags: [], chatTemplate: template, hasVisionConfig: vision, hasProjector: projector)170 }171 #expect(caps("org/chat", pipeline: "text-generation", template: "plain") == [.chat])172 #expect(caps("org/chat", template: "{% if tools %}") == [.chat, .tools])173 #expect(caps("org/chat", template: "{{ enable_thinking }}") == [.chat, .reasoning])174 #expect(caps("org/Qwen2.5-Coder-7B", template: "x") == [.chat, .code])175 #expect(caps("org/vlm", pipeline: "image-text-to-text") == [.chat, .vision])176 #expect(caps("org/gguf", projector: true) == [.chat, .vision])177 #expect(caps("org/DeepSeek-R1-Distill") == [.chat, .reasoning])178 #expect(caps("BAAI/bge-small-en-v1.5", pipeline: "feature-extraction") == [.embeddings])179 #expect(caps("org/all-MiniLM-L6-v2", pipeline: "sentence-similarity") == [.embeddings])180 #expect(caps("org/Qwen3-Embedding-0.6B-4bit-DWQ", pipeline: "text-generation", template: "x") == [.embeddings])181 }182}183184@Suite struct ModelHubResolverTests {185 @Test func mlxRepositoryYieldsOneCandidateWithExactFiles() throws {186 let model = try JSONDecoder().decode(HubModel.self, from: Data(mlxDetailJSON.utf8))187 let candidates = ModelMetadataResolver().candidates(for: model)188 #expect(candidates.count == 1)189 let c = try #require(candidates.first)190 #expect(c.descriptor.id == "mlx:mlx-community/Qwen3-4B-4bit")191 #expect(c.descriptor.backend == .mlx && c.descriptor.format == .mlx)192 #expect(c.descriptor.quantization == "4bit" && c.descriptor.architecture == "qwen3")193 #expect(c.descriptor.parameterCount == 4_022_468_096 && c.descriptor.parameterProvenance == .declared)194 #expect(c.descriptor.capabilities == [.chat, .tools, .reasoning])195 #expect(c.descriptor.license == "apache-2.0" && c.descriptor.revision == model.commit && c.descriptor.localPath == nil)196 // README and .gitattributes are not downloaded197 #expect(Set(c.files.map(\.path)) == ["added_tokens.json", "config.json", "merges.txt", "model.safetensors", "model.safetensors.index.json",198 "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", "vocab.json"])199 let expected: Int64 = 707 + 937 + 1_671_853 + 2_263_022_529 + 63_924 + 613 + 11_422_654 + 9_706 + 2_776_833200 #expect(c.descriptor.sizeBytes == expected && c.downloadBytes == expected)201 #expect(c.unsupportedReason == nil)202 #expect(DownloadRequest(candidate: c)?.files.count == 9)203 }204205 @Test func unsupportedArchitectureIsReportedNotHidden() throws {206 let model = try JSONDecoder().decode(HubModel.self, from: Data(mlxDetailJSON.utf8))207 let supported = ModelMetadataResolver(supportedMLXArchitectures: ["qwen3", "llama"]).mlxCandidate(for: model)208 #expect(supported?.unsupportedReason == nil)209 let unsupported = ModelMetadataResolver(supportedMLXArchitectures: ["llama"]).mlxCandidate(for: model)210 #expect(unsupported?.unsupportedReason?.contains("qwen3") == true)211 }212213 @Test func ggufRepositoryYieldsOneCandidatePerFileAndGroupsSplits() throws {214 let model = try JSONDecoder().decode(HubModel.self, from: Data(ggufDetailJSON.utf8))215 let resolver = ModelMetadataResolver()216 #expect(resolver.mlxCandidate(for: model) == nil)217 let candidates = resolver.candidates(for: model)218 // 3 single files + 1 complete split group; the orphan part is not offered219 #expect(candidates.map(\.descriptor.id) == [220 "gguf:unsloth/Qwen3-4B-GGUF:big-Q6_K-00001-of-00002.gguf", "gguf:unsloth/Qwen3-4B-GGUF:Qwen3-4B-Q4_K_M.gguf",221 "gguf:unsloth/Qwen3-4B-GGUF:Qwen3-4B-UD-Q4_K_XL.gguf", "gguf:unsloth/Qwen3-4B-GGUF:Qwen3-4B-Q8_0.gguf",222 ])223 let split = candidates[0]224 #expect(split.files.count == 2 && split.descriptor.sizeBytes == 150 && split.descriptor.name == "big-Q6_K")225 let q4 = candidates[1]226 #expect(q4.descriptor.quantization == "Q4_K_M" && q4.descriptor.contextLength == 40960)227 #expect(q4.descriptor.parameterCount == 4_022_468_096 && q4.descriptor.parameterProvenance == .declared)228 #expect(q4.descriptor.capabilities == [.chat, .tools] && q4.descriptor.fileName == "Qwen3-4B-Q4_K_M.gguf")229 #expect(q4.descriptor.architecture == "qwen3" && q4.projector == nil)230 #expect(ModelMetadataResolver(supportedGGUFArchitectures: ["llama"]).ggufCandidates(for: model).allSatisfy { $0.unsupportedReason != nil })231 }232233 @Test func ggufProjectorMakesVisionCandidates() throws {234 let json = #"{"id":"org/vlm-GGUF","tags":["gguf"],"pipeline_tag":"image-text-to-text","gated":"auto","siblings":[{"rfilename":"vlm-7B-Q4_K_M.gguf","size":10,"lfs":{"sha256":"a","size":10}},{"rfilename":"mmproj-vlm-f16.gguf","size":3,"lfs":{"sha256":"b","size":3}},{"rfilename":"mmproj-vlm-Q8_0.gguf","size":2,"lfs":{"sha256":"c","size":2}}]}"#235 let model = try JSONDecoder().decode(HubModel.self, from: Data(json.utf8))236 let candidates = ModelMetadataResolver().candidates(for: model)237 #expect(candidates.count == 1)238 let c = try #require(candidates.first)239 #expect(c.projector?.path == "mmproj-vlm-f16.gguf" && c.files.map(\.path) == ["vlm-7B-Q4_K_M.gguf", "mmproj-vlm-f16.gguf"])240 #expect(c.descriptor.capabilities.contains(.vision) && c.descriptor.sizeBytes == 13)241 #expect(c.descriptor.parameterCount == 7_000_000_000 && c.descriptor.parameterProvenance == .inferred)242 #expect(c.isGated && c.notes.contains { $0.contains("Gated") })243 }244245 @Test func plainSafetensorsRepositoriesAreOnlyOfferedForEmbedders() throws {246 let embedder = #"{"id":"BAAI/bge-small-en-v1.5","library_name":"sentence-transformers","pipeline_tag":"feature-extraction","tags":["sentence-transformers","safetensors","bert","license:mit"],"config":{"model_type":"bert"},"safetensors":{"total":33360512},"siblings":[{"rfilename":"config.json","size":743},{"rfilename":"model.safetensors","size":133466304,"lfs":{"sha256":"aa","size":133466304}},{"rfilename":"pytorch_model.bin","size":133508397,"lfs":{"sha256":"bb","size":133508397}},{"rfilename":"onnx/model.onnx","size":133093490,"lfs":{"sha256":"cc","size":133093490}},{"rfilename":"1_Pooling/config.json","size":190},{"rfilename":"tokenizer.json","size":711396},{"rfilename":"README.md","size":90000}]}"#247 let model = try JSONDecoder().decode(HubModel.self, from: Data(embedder.utf8))248 let c = try #require(ModelMetadataResolver().mlxCandidate(for: model))249 #expect(c.descriptor.capabilities == [.embeddings] && c.descriptor.isEmbeddingOnly && c.descriptor.license == "mit")250 #expect(Set(c.files.map(\.path)) == ["config.json", "model.safetensors", "1_Pooling/config.json", "tokenizer.json"])251 #expect(c.notes.contains { $0.contains("Not an MLX conversion") })252253 let plain = #"{"id":"Qwen/Qwen3-0.6B","library_name":"transformers","pipeline_tag":"text-generation","tags":["transformers","safetensors"],"config":{"model_type":"qwen3"},"siblings":[{"rfilename":"config.json","size":1},{"rfilename":"model.safetensors","size":2}]}"#254 #expect(ModelMetadataResolver().candidates(for: try JSONDecoder().decode(HubModel.self, from: Data(plain.utf8))).isEmpty)255 }256257 @Test func starterCatalogIsWellFormed() {258 #expect(StarterCatalog.entries.count >= 12)259 #expect(Set(StarterCatalog.entries.map(\.repoID)).count == StarterCatalog.entries.count)260 #expect(StarterCatalog.entries.allSatisfy { HuggingFaceAPI.isValidRepositoryID($0.repoID) && !$0.reason.isEmpty })261 for task in [TaskCategory.everyday, .reasoning, .coding, .vision, .embeddings] { #expect(!StarterCatalog.entries(for: task).isEmpty) }262 }263}264