SPB Git

Repository Public MIT

Governors.swift 17.8 KB 378 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraLocal/Sources/Governors.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90#if canImport(Darwin)91import Darwin92#endif9394// MARK: - Device facts9596/// Facts about the machine the app runs on, read from the system. Nothing here is estimated.97public struct DeviceFacts: Sendable, Hashable, Codable {98    public enum Platform: String, Sendable, Codable { case iOS, macOS, simulator }99    public var platform: Platform100    /// Hardware model identifier as the kernel reports it ("iPhone18,2", "Mac17,6").101    public var modelIdentifier: String102    public var osVersion: String103    public var physicalMemoryBytes: Int64104105    public init(platform: Platform, modelIdentifier: String, osVersion: String, physicalMemoryBytes: Int64) {106        self.platform = platform; self.modelIdentifier = modelIdentifier; self.osVersion = osVersion; self.physicalMemoryBytes = physicalMemoryBytes107    }108109    public static func current() -> DeviceFacts {110        #if targetEnvironment(simulator)111        let platform = Platform.simulator112        #elseif os(iOS)113        let platform = Platform.iOS114        #else115        let platform = Platform.macOS116        #endif117        let v = ProcessInfo.processInfo.operatingSystemVersion118        return DeviceFacts(platform: platform, modelIdentifier: readModelIdentifier(),119                           osVersion: "\(v.majorVersion).\(v.minorVersion).\(v.patchVersion)",120                           physicalMemoryBytes: Int64(ProcessInfo.processInfo.physicalMemory))121    }122123    static func readModelIdentifier() -> String {124        #if targetEnvironment(simulator)125        return ProcessInfo.processInfo.environment["SIMULATOR_MODEL_IDENTIFIER"] ?? "Simulator"126        #else127        #if os(macOS)128        let key = "hw.model"129        #else130        let key = "hw.machine"131        #endif132        var size = 0133        guard sysctlbyname(key, nil, &size, nil, 0) == 0, size > 0 else { return "unknown" }134        var buffer = [CChar](repeating: 0, count: size)135        guard sysctlbyname(key, &buffer, &size, nil, 0) == 0 else { return "unknown" }136        return String(decoding: buffer.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) }, as: UTF8.self)137        #endif138    }139}140141// MARK: - Memory142143/// One reading of the memory situation. Injected in tests; read from the system in the app.144public struct MemoryReading: Sendable, Hashable, Codable {145    public var physicalBytes: Int64146    /// Memory this process may still allocate before the system intervenes (`os_proc_available_memory` on iOS).147    /// nil where the system exposes no such figure (macOS).148    public var availableBytes: Int64?149    /// Current footprint of this process (`phys_footprint`), when readable.150    public var footprintBytes: Int64?151    public init(physicalBytes: Int64, availableBytes: Int64? = nil, footprintBytes: Int64? = nil) {152        self.physicalBytes = physicalBytes; self.availableBytes = availableBytes; self.footprintBytes = footprintBytes153    }154155    public static func current() -> MemoryReading {156        var available: Int64?157        #if os(iOS) && !targetEnvironment(simulator)158        let n = os_proc_available_memory()159        if n > 0 { available = Int64(n) }160        #endif161        return MemoryReading(physicalBytes: Int64(ProcessInfo.processInfo.physicalMemory), availableBytes: available, footprintBytes: processFootprint())162    }163164    static func processFootprint() -> Int64? {165        var info = task_vm_info_data_t()166        var count = mach_msg_type_number_t(MemoryLayout<task_vm_info_data_t>.size / MemoryLayout<integer_t>.size)167        let result = withUnsafeMutablePointer(to: &info) {168            $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) }169        }170        return result == KERN_SUCCESS ? Int64(info.phys_footprint) : nil171    }172}173174/// What the governor needs to know about a model to judge whether it fits.175public struct ModelMemoryProfile: Sendable, Hashable {176    /// Size of the weight files on disk; MLX maps them almost one-to-one into memory.177    public var weightsBytes: Int64178    /// KV-cache cost per context token, in bytes, when known from the architecture (layers × 2 × kvHeads × headDim × 2 bytes at fp16).179    public var kvBytesPerToken: Int64?180    /// Parameter count, used for a rough KV figure when the architecture numbers are unknown.181    public var parameterCount: Int64?182    /// The model's own maximum context.183    public var modelContextLength: Int?184    public init(weightsBytes: Int64, kvBytesPerToken: Int64? = nil, parameterCount: Int64? = nil, modelContextLength: Int? = nil) {185        self.weightsBytes = weightsBytes; self.kvBytesPerToken = kvBytesPerToken; self.parameterCount = parameterCount; self.modelContextLength = modelContextLength186    }187}188189/// A conservative judgement of whether a model fits. Always an estimate made before loading, never a guarantee.190public struct FitVerdict: Sendable, Hashable, Codable {191    public enum State: String, Sendable, Codable, CaseIterable {192        /// Needs at most 60 % of the budget.193        case comfortable194        /// Fits in the budget with little headroom; other apps may be evicted.195        case tight196        /// Exceeds the budget but not the device; the system will probably terminate the app.197        case unlikely198        /// Exceeds what this device could ever give one app.199        case tooLarge200    }201    public var state: State202    /// Weights + KV cache at the requested context + fixed runtime overhead.203    public var estimatedNeedBytes: Int64204    public var weightsBytes: Int64205    public var kvCacheBytes: Int64206    public var overheadBytes: Int64207    /// What the governor considers usable by one model on this device right now.208    public var budgetBytes: Int64209    public var physicalBytes: Int64210    public var availableBytes: Int64?211    public var contextTokens: Int212    /// Always true: the verdict is computed from file sizes and a KV-cache formula, not measured.213    public var isEstimate: Bool = true214215    public var mayLoad: Bool { state == .comfortable || state == .tight }216217    /// Sentence for the UI. States the numbers and that they are an estimate.218    public var explanation: String {219        let need = MemoryGovernor.format(estimatedNeedBytes), budget = MemoryGovernor.format(budgetBytes)220        let lead: String = switch state {221        case .comfortable: "Should run comfortably"222        case .tight: "Should run, with little memory to spare"223        case .unlikely: "Unlikely to run"224        case .tooLarge: "Too large for this device"225        }226        return "\(lead): about \(need) needed at \(contextTokens) tokens of context, \(budget) usable on this device. This is an estimate, not a guarantee."227    }228}229230/// Decides, from readings, whether a model fits and how much context this device should use. Pure: every input is a parameter.231public struct MemoryGovernor: Sendable {232    public var platform: DeviceFacts.Platform233    /// Fixed allowance for the runtime, tokenizer, Metal buffers and the app itself.234    public var overheadBytes: Int64 = 350 << 20235    /// Share of physical memory one model may use.236    public var physicalShare: Double237    /// Share of the currently available memory one model may use (iOS only).238    public var availableShare: Double = 0.85239240    public init(platform: DeviceFacts.Platform = DeviceFacts.current().platform) {241        self.platform = platform242        // An iPhone app is terminated near 50–60 % of physical memory (more with the increased-memory-limit entitlement, which is not assumed).243        physicalShare = platform == .macOS ? 0.70 : 0.50244    }245246    public func budget(_ reading: MemoryReading) -> Int64 {247        var budget = Int64(Double(reading.physicalBytes) * physicalShare)248        if let available = reading.availableBytes {249            // What the app already uses for a loaded model is not counted against the next one: callers unload first.250            budget = min(budget, Int64(Double(available) * availableShare))251        }252        return max(0, budget)253    }254255    public func kvCacheBytes(_ model: ModelMemoryProfile, context: Int) -> Int64 {256        if let perToken = model.kvBytesPerToken { return perToken * Int64(context) }257        // Without architecture numbers: small models measured here use 20–60 KB per token at fp16; scale with size and stay on the high side.258        let params = Double(model.parameterCount ?? max(1, model.weightsBytes * 2))   // 4-bit weights ≈ 0.5 byte per parameter259        let perToken = min(400_000, max(24_000, params / 1e9 * 70_000))260        return Int64(perToken) * Int64(context)261    }262263    public func verdict(for model: ModelMemoryProfile, context: Int, reading: MemoryReading) -> FitVerdict {264        let context = max(256, min(context, model.modelContextLength ?? context))265        let kv = kvCacheBytes(model, context: context)266        let need = model.weightsBytes + kv + overheadBytes267        let budget = budget(reading)268        let state: FitVerdict.State269        if need > Int64(Double(reading.physicalBytes) * 0.80) { state = .tooLarge }270        else if need > budget { state = .unlikely }271        else if Double(need) > Double(budget) * 0.60 { state = .tight }272        else { state = .comfortable }273        return FitVerdict(state: state, estimatedNeedBytes: need, weightsBytes: model.weightsBytes, kvCacheBytes: kv, overheadBytes: overheadBytes,274                          budgetBytes: budget, physicalBytes: reading.physicalBytes, availableBytes: reading.availableBytes, contextTokens: context)275    }276277    /// Largest context from a fixed ladder that keeps the model inside the budget; nil when even the smallest step does not fit.278    public func recommendedMaxContext(for model: ModelMemoryProfile, reading: MemoryReading) -> Int? {279        let ladder = [32_768, 16_384, 8_192, 4_096, 2_048, 1_024]280        let ceiling = model.modelContextLength ?? ladder[0]281        // Phones stay at 8k or less: beyond that, prompt evaluation time and heat dominate long before memory does.282        let cap = platform == .macOS ? 32_768 : 8_192283        for step in ladder where step <= min(ceiling, cap) {284            if verdict(for: model, context: step, reading: reading).mayLoad { return step }285        }286        if ceiling < 1_024, verdict(for: model, context: ceiling, reading: reading).mayLoad { return ceiling }287        return nil288    }289290    /// Default context for this device before any model is considered.291    public func defaultContext(reading: MemoryReading) -> Int {292        let gb = Double(reading.physicalBytes) / 1_073_741_824293        if platform == .macOS { return gb >= 32 ? 16_384 : 8_192 }294        return gb >= 11 ? 8_192 : gb >= 7 ? 4_096 : 2_048295    }296297    public static func format(_ bytes: Int64) -> String {298        let gb = Double(bytes) / 1_073_741_824299        if gb >= 1 { return String(format: "%.1f GB", gb) }300        return String(format: "%.0f MB", Double(bytes) / 1_048_576)301    }302}303304// MARK: - Thermal305306public enum ThermalLevel: String, Sendable, Codable, Hashable, CaseIterable, Comparable {307    case nominal, fair, serious, critical308    public init(_ state: ProcessInfo.ThermalState) {309        switch state {310        case .nominal: self = .nominal311        case .fair: self = .fair312        case .serious: self = .serious313        case .critical: self = .critical314        @unknown default: self = .fair315        }316    }317    private var rank: Int { switch self { case .nominal: 0; case .fair: 1; case .serious: 2; case .critical: 3 } }318    public static func < (a: ThermalLevel, b: ThermalLevel) -> Bool { a.rank < b.rank }319    public static func current() -> ThermalLevel { ThermalLevel(ProcessInfo.processInfo.thermalState) }320}321322/// What the app must do at a given thermal level.323public struct ThermalDecision: Sendable, Hashable {324    public var level: ThermalLevel325    public var allowsNewGeneration: Bool326    public var shouldUnload: Bool327    /// Multipliers applied to the requested limits (1 = unchanged).328    public var maxTokensFactor: Double329    public var contextFactor: Double330    /// Text for the UI; nil when there is nothing to say.331    public var warning: String?332333    /// Applies the reduction to request parameters. Never raises a limit.334    public func limit(maxTokens: Int?, context: Int?) -> (maxTokens: Int?, context: Int?) {335        guard maxTokensFactor < 1 || contextFactor < 1 else { return (maxTokens, context) }336        let tokens = Int(Double(maxTokens ?? ThermalGovernor.defaultMaxTokens) * maxTokensFactor)337        let ctx = context.map { max(1_024, Int(Double($0) * contextFactor)) }338        return (max(64, tokens), ctx)339    }340}341342/// Thermal policy: serious → shorter answers and smaller context, with a warning; critical → no new generation, unload. Pure.343public struct ThermalGovernor: Sendable {344    public static let defaultMaxTokens = 1_024345    public var seriousMaxTokensFactor = 0.5346    public var seriousContextFactor = 0.5347    public init() {}348349    public func decision(for level: ThermalLevel) -> ThermalDecision {350        switch level {351        case .nominal, .fair:352            ThermalDecision(level: level, allowsNewGeneration: true, shouldUnload: false, maxTokensFactor: 1, contextFactor: 1, warning: nil)353        case .serious:354            ThermalDecision(level: level, allowsNewGeneration: true, shouldUnload: false, maxTokensFactor: seriousMaxTokensFactor, contextFactor: seriousContextFactor,355                            warning: "The device is hot. Answers are shortened and the context is reduced until it cools down.")356        case .critical:357            ThermalDecision(level: level, allowsNewGeneration: false, shouldUnload: true, maxTokensFactor: 0, contextFactor: 0,358                            warning: "The device is too hot to run a model. The model was unloaded; try again when it has cooled down.")359        }360    }361362    /// Stream of thermal levels: the current one first, then every change reported by the system.363    public static func levels() -> AsyncStream<ThermalLevel> {364        AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in365            continuation.yield(.current())366            let box = ObserverBox(NotificationCenter.default.addObserver(forName: ProcessInfo.thermalStateDidChangeNotification, object: nil, queue: nil) { _ in367                continuation.yield(.current())368            })369            continuation.onTermination = { _ in NotificationCenter.default.removeObserver(box.token) }370        }371    }372}373374final class ObserverBox: @unchecked Sendable {375    let token: NSObjectProtocol376    init(_ token: NSObjectProtocol) { self.token = token }377}378
Files main
Menu

Theme