SPB Git

Repository Public MIT

HuggingFaceResolver.swift 9.6 KB 193 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraLocal/Sources/HuggingFaceResolver.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation9091/// A repository pinned to one commit, as the public Hugging Face API describes it. No API key is used.92public struct HubSnapshot: Sendable, Hashable, Codable {93    public struct File: Sendable, Hashable, Codable {94        public var path: String95        public var size: Int6496        /// SHA-256 published for files stored in LFS.97        public var sha256: String?98        /// Git blob id of files stored in git itself.99        public var gitBlobSHA1: String?100    }101    public var repo: String102    /// Full commit hash the file list belongs to.103    public var revision: String104    public var files: [File]105    /// License tag of the model card, when the repository declares one.106    public var license: String?107    public var tags: [String]108    public var pipelineTag: String?109    public var baseModels: [String]110    /// Parameter count reported by the Hub from the safetensors headers. For quantized MLX weights this counts packed tensors, not the original model's parameters.111    public var safetensorsParameters: Int64?112    public var lastModified: String?113114    public var totalBytes: Int64 { files.reduce(0) { $0 + $1.size } }115116    /// Files a runtime needs: everything except documentation, repository metadata and pictures.117    public var runtimeFiles: [File] {118        files.filter { f in119            let name = f.path.lowercased()120            if name.hasPrefix(".") || name.contains("/.") { return false }121            return ![".md", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".mp4", ".pdf"].contains { name.hasSuffix($0) }122        }123    }124125    public func plan(id: String, card: LocalModelCard? = nil) -> DownloadPlan {126        DownloadPlan(id: id, files: runtimeFiles.map { .init(path: $0.path, size: $0.size, sha256: $0.sha256, gitBlobSHA1: $0.gitBlobSHA1) },127                     sources: [URL(string: LocalModelCard.huggingFaceBase(repo: repo, revision: revision))!], card: card)128    }129}130131/// Resolves a repository's current commit and exact file list from the Hub.132public struct HuggingFaceResolver: Sendable {133    let transport: any HTTPTransport134    public var endpoint: URL135    public init(transport: any HTTPTransport, endpoint: URL = URL(string: "https://huggingface.co")!) { self.transport = transport; self.endpoint = endpoint }136137    public func snapshot(repo: String, revision: String? = nil) async throws -> HubSnapshot {138        guard repo.split(separator: "/").count == 2, !repo.contains(".."), repo.allSatisfy({ $0.isLetter || $0.isNumber || "-_./".contains($0) }) else {139            throw AstraError.invalidRequest("“\(repo)” is not a repository name.")140        }141        var infoPath = "api/models/\(repo)"142        if let revision { infoPath += "/revision/\(revision)" }143        let info = try await get(infoPath, repo: repo)144        guard let sha = info["sha"]?.stringValue, ModelPath.isHex(sha, count: 40) else { throw AstraError.decoding("the Hub did not return a commit for \(repo)") }145        let card = info["cardData"]146        var baseModels: [String] = []147        if let one = card?["base_model"]?.stringValue { baseModels = [one] } else if let many = card?["base_model"]?.arrayValue { baseModels = many.compactMap(\.stringValue) }148149        var files: [HubSnapshot.File] = []150        var cursor: String? = "api/models/\(repo)/tree/\(sha)?recursive=true&limit=1000"151        var pages = 0152        while let path = cursor, pages < 20 {153            pages += 1154            let (value, next) = try await getPage(path, repo: repo)155            for item in value.arrayValue ?? [] where item["type"]?.stringValue == "file" {156                guard let p = item["path"]?.stringValue, let size = item["lfs"]?["size"]?.intValue ?? item["size"]?.intValue else { continue }157                let lfs = item["lfs"]?["oid"]?.stringValue158                files.append(.init(path: p, size: Int64(size), sha256: lfs, gitBlobSHA1: lfs == nil ? item["oid"]?.stringValue : nil))159            }160            cursor = next161        }162        return HubSnapshot(repo: repo, revision: sha, files: files.sorted { $0.path < $1.path },163                           license: card?["license"]?.stringValue ?? info["tags"]?.arrayValue?.compactMap(\.stringValue).first { $0.hasPrefix("license:") }.map { String($0.dropFirst(8)) },164                           tags: info["tags"]?.arrayValue?.compactMap(\.stringValue) ?? [], pipelineTag: info["pipeline_tag"]?.stringValue,165                           baseModels: baseModels, safetensorsParameters: info["safetensors"]?["total"]?.intValue.map(Int64.init),166                           lastModified: info["lastModified"]?.stringValue)167    }168169    private func get(_ path: String, repo: String) async throws -> JSONValue { try await getPage(path, repo: repo).0 }170171    private func getPage(_ path: String, repo: String) async throws -> (JSONValue, String?) {172        guard let url = URL(string: path, relativeTo: endpoint) else { throw AstraError.invalidRequest("Malformed Hub URL.") }173        var request = URLRequest(url: url)174        request.timeoutInterval = 30175        request.setValue("application/json", forHTTPHeaderField: "Accept")176        let (data, response) = try await transport.send(request)177        switch response.statusCode {178        case 200: break179        case 401, 404:   // without a token the Hub answers 401 for a repository that does not exist (or is private)180            throw AstraError.local("Hugging Face has no public repository named “\(repo)” (HTTP \(response.statusCode)).")181        default: throw AstraError.server(status: response.statusCode, message: "The Hugging Face API answered HTTP \(response.statusCode) for \(repo).")182        }183        guard let value = JSONValue.parse(data) else { throw AstraError.decoding("the Hub returned something that is not JSON") }184        // Link: <https://huggingface.co/api/...>; rel="next"185        var next: String?186        if let link = response.header("Link"), let range = link.range(of: #"<([^>]+)>;\s*rel="next""#, options: .regularExpression) {187            let raw = link[range]188            if let open = raw.firstIndex(of: "<"), let close = raw.firstIndex(of: ">") { next = String(raw[raw.index(after: open)..<close]) }189        }190        return (value, next)191    }192}193
Files main
Menu

Theme