Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Packages/AstraLocal/Sources/HuggingFaceResolver.swift10// Module : Packages11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation9091/// A repository pinned to one commit, as the public Hugging Face API describes it. No API key is used.92public struct HubSnapshot: Sendable, Hashable, Codable {93 public struct File: Sendable, Hashable, Codable {94 public var path: String95 public var size: Int6496 /// SHA-256 published for files stored in LFS.97 public var sha256: String?98 /// Git blob id of files stored in git itself.99 public var gitBlobSHA1: String?100 }101 public var repo: String102 /// Full commit hash the file list belongs to.103 public var revision: String104 public var files: [File]105 /// License tag of the model card, when the repository declares one.106 public var license: String?107 public var tags: [String]108 public var pipelineTag: String?109 public var baseModels: [String]110 /// Parameter count reported by the Hub from the safetensors headers. For quantized MLX weights this counts packed tensors, not the original model's parameters.111 public var safetensorsParameters: Int64?112 public var lastModified: String?113114 public var totalBytes: Int64 { files.reduce(0) { $0 + $1.size } }115116 /// Files a runtime needs: everything except documentation, repository metadata and pictures.117 public var runtimeFiles: [File] {118 files.filter { f in119 let name = f.path.lowercased()120 if name.hasPrefix(".") || name.contains("/.") { return false }121 return ![".md", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".mp4", ".pdf"].contains { name.hasSuffix($0) }122 }123 }124125 public func plan(id: String, card: LocalModelCard? = nil) -> DownloadPlan {126 DownloadPlan(id: id, files: runtimeFiles.map { .init(path: $0.path, size: $0.size, sha256: $0.sha256, gitBlobSHA1: $0.gitBlobSHA1) },127 sources: [URL(string: LocalModelCard.huggingFaceBase(repo: repo, revision: revision))!], card: card)128 }129}130131/// Resolves a repository's current commit and exact file list from the Hub.132public struct HuggingFaceResolver: Sendable {133 let transport: any HTTPTransport134 public var endpoint: URL135 public init(transport: any HTTPTransport, endpoint: URL = URL(string: "https://huggingface.co")!) { self.transport = transport; self.endpoint = endpoint }136137 public func snapshot(repo: String, revision: String? = nil) async throws -> HubSnapshot {138 guard repo.split(separator: "/").count == 2, !repo.contains(".."), repo.allSatisfy({ $0.isLetter || $0.isNumber || "-_./".contains($0) }) else {139 throw AstraError.invalidRequest("“\(repo)” is not a repository name.")140 }141 var infoPath = "api/models/\(repo)"142 if let revision { infoPath += "/revision/\(revision)" }143 let info = try await get(infoPath, repo: repo)144 guard let sha = info["sha"]?.stringValue, ModelPath.isHex(sha, count: 40) else { throw AstraError.decoding("the Hub did not return a commit for \(repo)") }145 let card = info["cardData"]146 var baseModels: [String] = []147 if let one = card?["base_model"]?.stringValue { baseModels = [one] } else if let many = card?["base_model"]?.arrayValue { baseModels = many.compactMap(\.stringValue) }148149 var files: [HubSnapshot.File] = []150 var cursor: String? = "api/models/\(repo)/tree/\(sha)?recursive=true&limit=1000"151 var pages = 0152 while let path = cursor, pages < 20 {153 pages += 1154 let (value, next) = try await getPage(path, repo: repo)155 for item in value.arrayValue ?? [] where item["type"]?.stringValue == "file" {156 guard let p = item["path"]?.stringValue, let size = item["lfs"]?["size"]?.intValue ?? item["size"]?.intValue else { continue }157 let lfs = item["lfs"]?["oid"]?.stringValue158 files.append(.init(path: p, size: Int64(size), sha256: lfs, gitBlobSHA1: lfs == nil ? item["oid"]?.stringValue : nil))159 }160 cursor = next161 }162 return HubSnapshot(repo: repo, revision: sha, files: files.sorted { $0.path < $1.path },163 license: card?["license"]?.stringValue ?? info["tags"]?.arrayValue?.compactMap(\.stringValue).first { $0.hasPrefix("license:") }.map { String($0.dropFirst(8)) },164 tags: info["tags"]?.arrayValue?.compactMap(\.stringValue) ?? [], pipelineTag: info["pipeline_tag"]?.stringValue,165 baseModels: baseModels, safetensorsParameters: info["safetensors"]?["total"]?.intValue.map(Int64.init),166 lastModified: info["lastModified"]?.stringValue)167 }168169 private func get(_ path: String, repo: String) async throws -> JSONValue { try await getPage(path, repo: repo).0 }170171 private func getPage(_ path: String, repo: String) async throws -> (JSONValue, String?) {172 guard let url = URL(string: path, relativeTo: endpoint) else { throw AstraError.invalidRequest("Malformed Hub URL.") }173 var request = URLRequest(url: url)174 request.timeoutInterval = 30175 request.setValue("application/json", forHTTPHeaderField: "Accept")176 let (data, response) = try await transport.send(request)177 switch response.statusCode {178 case 200: break179 case 401, 404: // without a token the Hub answers 401 for a repository that does not exist (or is private)180 throw AstraError.local("Hugging Face has no public repository named “\(repo)” (HTTP \(response.statusCode)).")181 default: throw AstraError.server(status: response.statusCode, message: "The Hugging Face API answered HTTP \(response.statusCode) for \(repo).")182 }183 guard let value = JSONValue.parse(data) else { throw AstraError.decoding("the Hub returned something that is not JSON") }184 // Link: <https://huggingface.co/api/...>; rel="next"185 var next: String?186 if let link = response.header("Link"), let range = link.range(of: #"<([^>]+)>;\s*rel="next""#, options: .regularExpression) {187 let raw = link[range]188 if let open = raw.firstIndex(of: "<"), let close = raw.firstIndex(of: ">") { next = String(raw[raw.index(after: open)..<close]) }189 }190 return (value, next)191 }192}193