SPB Git

Repository Public MIT

ModelStore.swift 14.1 KB 301 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraLocal/Sources/ModelStore.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import CryptoKit90import Foundation9192/// One model present on disk, complete or not.93public struct InstalledModel: Sendable, Hashable, Identifiable {94    public var card: LocalModelCard95    public var directory: URL96    /// Bytes really allocated on disk for this model, partial files included.97    public var bytesOnDisk: Int6498    public var installedAt: Date?99    /// False while a download is unfinished: the model must not be loaded.100    public var isComplete: Bool101    public var id: String { card.id }102}103104/// On-disk layout of downloaded models: `<root>/<id>/<repository files>` plus a private `<root>/<id>/.astra/` folder105/// holding the card and the download state. The root is excluded from iCloud and device backups.106public struct ModelStore: Sendable {107    public let root: URL108    static let metaFolder = ".astra"109    static let cardFile = "card.json"110    static let installedFile = "installed.json"111    static let downloadFile = "download.json"112    static let partialSuffix = ".partial"113114    public init(root: URL) { self.root = root.standardizedFileURL }115116    /// `Application Support/Models` of this app.117    public static func defaultRoot() throws -> URL {118        try FileManager.default.url(for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true)119            .appendingPathComponent("Models", isDirectory: true)120    }121122    struct InstallRecord: Codable { var installedAt: Date; var revision: String }123124    /// Creates the root and marks it as excluded from backups. Safe to call repeatedly.125    public func prepare() throws {126        try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)127        Self.excludeFromBackup(root)128    }129130    static func excludeFromBackup(_ url: URL) {131        var u = url; var values = URLResourceValues(); values.isExcludedFromBackup = true132        try? u.setResourceValues(values)133    }134135    public func isExcludedFromBackup() -> Bool {136        (try? root.resourceValues(forKeys: [.isExcludedFromBackupKey]).isExcludedFromBackup) ?? false137    }138139    /// Directory of a model. Throws for an id that could escape the root.140    public func directory(for id: String) throws -> URL {141        guard ModelPath.isSafeID(id) else { throw AstraError.local("“\(id)” is not a valid model identifier.") }142        return root.appendingPathComponent(id, isDirectory: true)143    }144145    func meta(_ id: String) throws -> URL { try directory(for: id).appendingPathComponent(Self.metaFolder, isDirectory: true) }146147    /// Location of one model file, guaranteed to be inside the model directory (no traversal, no symlink escape).148    public func fileURL(modelID: String, path: String) throws -> URL {149        guard ModelPath.isSafe(path) else { throw AstraError.local("The file path “\(path)” is not allowed.") }150        let dir = try directory(for: modelID)151        let url = dir.appendingPathComponent(path).standardizedFileURL152        guard url.path.hasPrefix(dir.path + "/") else { throw AstraError.local("The file path “\(path)” leaves the model folder.") }153        // No component below the model folder may be a symbolic link: a link planted there would redirect the write elsewhere.154        var walk = dir155        for part in path.split(separator: "/") {156            walk.appendPathComponent(String(part))157            if (try? FileManager.default.attributesOfItem(atPath: walk.path))?[.type] as? FileAttributeType == .typeSymbolicLink {158                throw AstraError.local("“\(path)” goes through a symbolic link; it was not written.")159            }160        }161        return url162    }163164    // MARK: Records165166    func writeCard(_ card: LocalModelCard) throws {167        let dir = try meta(card.id)168        try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)169        Self.excludeFromBackup(try directory(for: card.id))170        let e = JSONEncoder(); e.outputFormatting = [.sortedKeys]171        try e.encode(card).write(to: dir.appendingPathComponent(Self.cardFile), options: .atomic)172    }173174    func markInstalled(_ card: LocalModelCard, at date: Date = Date()) throws {175        try writeCard(card)176        let e = JSONEncoder(); e.dateEncodingStrategy = .iso8601177        try e.encode(InstallRecord(installedAt: date, revision: card.revision)).write(to: try meta(card.id).appendingPathComponent(Self.installedFile), options: .atomic)178        try? FileManager.default.removeItem(at: try meta(card.id).appendingPathComponent(Self.downloadFile))179    }180181    public func card(id: String) -> LocalModelCard? {182        guard let dir = try? meta(id), let data = try? Data(contentsOf: dir.appendingPathComponent(Self.cardFile)) else { return nil }183        return try? JSONDecoder().decode(LocalModelCard.self, from: data)184    }185186    func installRecord(id: String) -> InstallRecord? {187        guard let dir = try? meta(id), let data = try? Data(contentsOf: dir.appendingPathComponent(Self.installedFile)) else { return nil }188        let d = JSONDecoder(); d.dateDecodingStrategy = .iso8601189        return try? d.decode(InstallRecord.self, from: data)190    }191192    /// True when the model finished downloading and every listed file is present with its declared size.193    public func isInstalled(_ id: String) -> Bool {194        guard let card = card(id: id), installRecord(id: id) != nil, let dir = try? directory(for: id) else { return false }195        return card.files.allSatisfy { Self.fileSize(dir.appendingPathComponent($0.path)) == $0.size }196    }197198    /// Directory to hand to the runtime; nil until the model is completely installed.199    public func loadableDirectory(for id: String) -> URL? { isInstalled(id) ? try? directory(for: id) : nil }200201    // MARK: Listing and usage202203    /// Every model folder with a card: installed models and unfinished downloads, with the bytes they really occupy.204    public func models() -> [InstalledModel] {205        let fm = FileManager.default206        guard let names = try? fm.contentsOfDirectory(atPath: root.path) else { return [] }207        return names.sorted().compactMap { name in208            guard ModelPath.isSafeID(name), let card = card(id: name), card.id == name, let dir = try? directory(for: name) else { return nil }209            return InstalledModel(card: card, directory: dir, bytesOnDisk: Self.allocatedBytes(dir), installedAt: installRecord(id: name)?.installedAt,210                                  isComplete: isInstalled(name))211        }212    }213214    public func installed() -> [InstalledModel] { models().filter(\.isComplete) }215216    /// Bytes used by one model folder, including partial downloads and metadata.217    public func bytesOnDisk(id: String) -> Int64 { (try? directory(for: id)).map(Self.allocatedBytes) ?? 0 }218219    /// Bytes used by everything under the root, including folders the app does not recognise.220    public func totalBytesOnDisk() -> Int64 { Self.allocatedBytes(root) }221222    /// Free space the system will let the app use on the volume holding the models.223    public func availableDiskBytes() -> Int64? {224        var url = root225        while !FileManager.default.fileExists(atPath: url.path), url.path != "/" { url.deleteLastPathComponent() }226        if let v = try? url.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]).volumeAvailableCapacityForImportantUsage, v > 0 { return v }227        return (try? url.resourceValues(forKeys: [.volumeAvailableCapacityKey]).volumeAvailableCapacity).map(Int64.init)228    }229230    // MARK: Removal231232    /// Removes a model completely: weights, partial files, card and download state. Returns the bytes freed.233    @discardableResult234    public func remove(id: String) throws -> Int64 {235        let dir = try directory(for: id)236        guard FileManager.default.fileExists(atPath: dir.path) else { return 0 }237        let freed = Self.allocatedBytes(dir)238        try FileManager.default.removeItem(at: dir)239        return freed240    }241242    /// Removes folders under the root that hold no card (leftovers of a crash before the first write). Returns the bytes freed.243    @discardableResult244    public func removeOrphans() -> Int64 {245        let fm = FileManager.default; var freed: Int64 = 0246        for name in (try? fm.contentsOfDirectory(atPath: root.path)) ?? [] where name != ".DS_Store" {247            let url = root.appendingPathComponent(name)248            if ModelPath.isSafeID(name), card(id: name) != nil { continue }249            freed += Self.allocatedBytes(url)250            try? fm.removeItem(at: url)251        }252        return freed253    }254255    // MARK: Helpers256257    /// Size of a regular file, read fresh from the file system (URL resource values are cached per URL object and go stale while a file grows).258    static func fileSize(_ url: URL) -> Int64? {259        guard let attributes = try? FileManager.default.attributesOfItem(atPath: url.path), attributes[.type] as? FileAttributeType == .typeRegular else { return nil }260        return (attributes[.size] as? NSNumber)?.int64Value261    }262263    static func allocatedBytes(_ url: URL) -> Int64 {264        let keys: [URLResourceKey] = [.isRegularFileKey, .totalFileAllocatedSizeKey, .fileAllocatedSizeKey]265        guard let walker = FileManager.default.enumerator(at: url, includingPropertiesForKeys: keys, options: []) else { return 0 }266        var total: Int64 = 0267        for case let file as URL in walker {268            guard let v = try? file.resourceValues(forKeys: Set(keys)), v.isRegularFile == true else { continue }269            total += Int64(v.totalFileAllocatedSize ?? v.fileAllocatedSize ?? 0)270        }271        return total272    }273}274275/// Streaming digests of files on disk: constant memory whatever the file size.276public enum FileDigest {277    static let chunk = 4 << 20278279    public static func sha256(of url: URL, shouldContinue: () -> Bool = { true }) throws -> String {280        let handle = try FileHandle(forReadingFrom: url)281        defer { try? handle.close() }282        var hasher = SHA256()283        while let data = try handle.read(upToCount: chunk), !data.isEmpty {284            guard shouldContinue() else { throw CancellationError() }285            hasher.update(data: data)286        }287        return hasher.finalize().map { String(format: "%02x", $0) }.joined()288    }289290    /// Git blob object id (SHA-1 of "blob <size>\0" + content): what the Hugging Face API publishes for files that are not stored in LFS.291    public static func gitBlobSHA1(of url: URL) throws -> String {292        let size = ModelStore.fileSize(url) ?? 0293        let handle = try FileHandle(forReadingFrom: url)294        defer { try? handle.close() }295        var hasher = Insecure.SHA1()296        hasher.update(data: Data("blob \(size)\0".utf8))297        while let data = try handle.read(upToCount: chunk), !data.isEmpty { hasher.update(data: data) }298        return hasher.finalize().map { String(format: "%02x", $0) }.joined()299    }300}301
Files main
Menu

Theme