Repository Public MIT
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX Astra9// File : Packages/AstraLocal/Sources/Governors.swift10// Module : Packages11// Repository : https://git.spboucher.ai/orix-astra12// Created : 2026-11-2113// Last Modified : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author : Simon-Pierre Boucher20// Position : Professor of Finance21// Institution : Université du Québec en Outaouais (UQO)22// Department : Département des sciences administratives23// Organization : SP Boucher24//25// Website : https://spboucher.ai26// Git : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General : info@spboucher.ai33// Professional : contact@spboucher.ai34// Direct : simon@spboucher.ai35// Research : research@spboucher.ai36// Development : dev@spboucher.ai37// Support : support@spboucher.ai38// Security/Admin : admin@spboucher.ai39// Notifications : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification : Public66// Distribution : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License : Proprietary, all rights reserved (see license file)80// License File : LICENSE81// Maintainer : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation90#if canImport(Darwin)91import Darwin92#endif9394// MARK: - Device facts9596/// Facts about the machine the app runs on, read from the system. Nothing here is estimated.97public struct DeviceFacts: Sendable, Hashable, Codable {98 public enum Platform: String, Sendable, Codable { case iOS, macOS, simulator }99 public var platform: Platform100 /// Hardware model identifier as the kernel reports it ("iPhone18,2", "Mac17,6").101 public var modelIdentifier: String102 public var osVersion: String103 public var physicalMemoryBytes: Int64104105 public init(platform: Platform, modelIdentifier: String, osVersion: String, physicalMemoryBytes: Int64) {106 self.platform = platform; self.modelIdentifier = modelIdentifier; self.osVersion = osVersion; self.physicalMemoryBytes = physicalMemoryBytes107 }108109 public static func current() -> DeviceFacts {110 #if targetEnvironment(simulator)111 let platform = Platform.simulator112 #elseif os(iOS)113 let platform = Platform.iOS114 #else115 let platform = Platform.macOS116 #endif117 let v = ProcessInfo.processInfo.operatingSystemVersion118 return DeviceFacts(platform: platform, modelIdentifier: readModelIdentifier(),119 osVersion: "\(v.majorVersion).\(v.minorVersion).\(v.patchVersion)",120 physicalMemoryBytes: Int64(ProcessInfo.processInfo.physicalMemory))121 }122123 static func readModelIdentifier() -> String {124 #if targetEnvironment(simulator)125 return ProcessInfo.processInfo.environment["SIMULATOR_MODEL_IDENTIFIER"] ?? "Simulator"126 #else127 #if os(macOS)128 let key = "hw.model"129 #else130 let key = "hw.machine"131 #endif132 var size = 0133 guard sysctlbyname(key, nil, &size, nil, 0) == 0, size > 0 else { return "unknown" }134 var buffer = [CChar](repeating: 0, count: size)135 guard sysctlbyname(key, &buffer, &size, nil, 0) == 0 else { return "unknown" }136 return String(decoding: buffer.prefix { $0 != 0 }.map { UInt8(bitPattern: $0) }, as: UTF8.self)137 #endif138 }139}140141// MARK: - Memory142143/// One reading of the memory situation. Injected in tests; read from the system in the app.144public struct MemoryReading: Sendable, Hashable, Codable {145 public var physicalBytes: Int64146 /// Memory this process may still allocate before the system intervenes (`os_proc_available_memory` on iOS).147 /// nil where the system exposes no such figure (macOS).148 public var availableBytes: Int64?149 /// Current footprint of this process (`phys_footprint`), when readable.150 public var footprintBytes: Int64?151 public init(physicalBytes: Int64, availableBytes: Int64? = nil, footprintBytes: Int64? = nil) {152 self.physicalBytes = physicalBytes; self.availableBytes = availableBytes; self.footprintBytes = footprintBytes153 }154155 public static func current() -> MemoryReading {156 var available: Int64?157 #if os(iOS) && !targetEnvironment(simulator)158 let n = os_proc_available_memory()159 if n > 0 { available = Int64(n) }160 #endif161 return MemoryReading(physicalBytes: Int64(ProcessInfo.processInfo.physicalMemory), availableBytes: available, footprintBytes: processFootprint())162 }163164 static func processFootprint() -> Int64? {165 var info = task_vm_info_data_t()166 var count = mach_msg_type_number_t(MemoryLayout<task_vm_info_data_t>.size / MemoryLayout<integer_t>.size)167 let result = withUnsafeMutablePointer(to: &info) {168 $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) }169 }170 return result == KERN_SUCCESS ? Int64(info.phys_footprint) : nil171 }172}173174/// What the governor needs to know about a model to judge whether it fits.175public struct ModelMemoryProfile: Sendable, Hashable {176 /// Size of the weight files on disk; MLX maps them almost one-to-one into memory.177 public var weightsBytes: Int64178 /// KV-cache cost per context token, in bytes, when known from the architecture (layers × 2 × kvHeads × headDim × 2 bytes at fp16).179 public var kvBytesPerToken: Int64?180 /// Parameter count, used for a rough KV figure when the architecture numbers are unknown.181 public var parameterCount: Int64?182 /// The model's own maximum context.183 public var modelContextLength: Int?184 public init(weightsBytes: Int64, kvBytesPerToken: Int64? = nil, parameterCount: Int64? = nil, modelContextLength: Int? = nil) {185 self.weightsBytes = weightsBytes; self.kvBytesPerToken = kvBytesPerToken; self.parameterCount = parameterCount; self.modelContextLength = modelContextLength186 }187}188189/// A conservative judgement of whether a model fits. Always an estimate made before loading, never a guarantee.190public struct FitVerdict: Sendable, Hashable, Codable {191 public enum State: String, Sendable, Codable, CaseIterable {192 /// Needs at most 60 % of the budget.193 case comfortable194 /// Fits in the budget with little headroom; other apps may be evicted.195 case tight196 /// Exceeds the budget but not the device; the system will probably terminate the app.197 case unlikely198 /// Exceeds what this device could ever give one app.199 case tooLarge200 }201 public var state: State202 /// Weights + KV cache at the requested context + fixed runtime overhead.203 public var estimatedNeedBytes: Int64204 public var weightsBytes: Int64205 public var kvCacheBytes: Int64206 public var overheadBytes: Int64207 /// What the governor considers usable by one model on this device right now.208 public var budgetBytes: Int64209 public var physicalBytes: Int64210 public var availableBytes: Int64?211 public var contextTokens: Int212 /// Always true: the verdict is computed from file sizes and a KV-cache formula, not measured.213 public var isEstimate: Bool = true214215 public var mayLoad: Bool { state == .comfortable || state == .tight }216217 /// Sentence for the UI. States the numbers and that they are an estimate.218 public var explanation: String {219 let need = MemoryGovernor.format(estimatedNeedBytes), budget = MemoryGovernor.format(budgetBytes)220 let lead: String = switch state {221 case .comfortable: "Should run comfortably"222 case .tight: "Should run, with little memory to spare"223 case .unlikely: "Unlikely to run"224 case .tooLarge: "Too large for this device"225 }226 return "\(lead): about \(need) needed at \(contextTokens) tokens of context, \(budget) usable on this device. This is an estimate, not a guarantee."227 }228}229230/// Decides, from readings, whether a model fits and how much context this device should use. Pure: every input is a parameter.231public struct MemoryGovernor: Sendable {232 public var platform: DeviceFacts.Platform233 /// Fixed allowance for the runtime, tokenizer, Metal buffers and the app itself.234 public var overheadBytes: Int64 = 350 << 20235 /// Share of physical memory one model may use.236 public var physicalShare: Double237 /// Share of the currently available memory one model may use (iOS only).238 public var availableShare: Double = 0.85239240 public init(platform: DeviceFacts.Platform = DeviceFacts.current().platform) {241 self.platform = platform242 // An iPhone app is terminated near 50–60 % of physical memory (more with the increased-memory-limit entitlement, which is not assumed).243 physicalShare = platform == .macOS ? 0.70 : 0.50244 }245246 public func budget(_ reading: MemoryReading) -> Int64 {247 var budget = Int64(Double(reading.physicalBytes) * physicalShare)248 if let available = reading.availableBytes {249 // What the app already uses for a loaded model is not counted against the next one: callers unload first.250 budget = min(budget, Int64(Double(available) * availableShare))251 }252 return max(0, budget)253 }254255 public func kvCacheBytes(_ model: ModelMemoryProfile, context: Int) -> Int64 {256 if let perToken = model.kvBytesPerToken { return perToken * Int64(context) }257 // Without architecture numbers: small models measured here use 20–60 KB per token at fp16; scale with size and stay on the high side.258 let params = Double(model.parameterCount ?? max(1, model.weightsBytes * 2)) // 4-bit weights ≈ 0.5 byte per parameter259 let perToken = min(400_000, max(24_000, params / 1e9 * 70_000))260 return Int64(perToken) * Int64(context)261 }262263 public func verdict(for model: ModelMemoryProfile, context: Int, reading: MemoryReading) -> FitVerdict {264 let context = max(256, min(context, model.modelContextLength ?? context))265 let kv = kvCacheBytes(model, context: context)266 let need = model.weightsBytes + kv + overheadBytes267 let budget = budget(reading)268 let state: FitVerdict.State269 if need > Int64(Double(reading.physicalBytes) * 0.80) { state = .tooLarge }270 else if need > budget { state = .unlikely }271 else if Double(need) > Double(budget) * 0.60 { state = .tight }272 else { state = .comfortable }273 return FitVerdict(state: state, estimatedNeedBytes: need, weightsBytes: model.weightsBytes, kvCacheBytes: kv, overheadBytes: overheadBytes,274 budgetBytes: budget, physicalBytes: reading.physicalBytes, availableBytes: reading.availableBytes, contextTokens: context)275 }276277 /// Largest context from a fixed ladder that keeps the model inside the budget; nil when even the smallest step does not fit.278 public func recommendedMaxContext(for model: ModelMemoryProfile, reading: MemoryReading) -> Int? {279 let ladder = [32_768, 16_384, 8_192, 4_096, 2_048, 1_024]280 let ceiling = model.modelContextLength ?? ladder[0]281 // Phones stay at 8k or less: beyond that, prompt evaluation time and heat dominate long before memory does.282 let cap = platform == .macOS ? 32_768 : 8_192283 for step in ladder where step <= min(ceiling, cap) {284 if verdict(for: model, context: step, reading: reading).mayLoad { return step }285 }286 if ceiling < 1_024, verdict(for: model, context: ceiling, reading: reading).mayLoad { return ceiling }287 return nil288 }289290 /// Default context for this device before any model is considered.291 public func defaultContext(reading: MemoryReading) -> Int {292 let gb = Double(reading.physicalBytes) / 1_073_741_824293 if platform == .macOS { return gb >= 32 ? 16_384 : 8_192 }294 return gb >= 11 ? 8_192 : gb >= 7 ? 4_096 : 2_048295 }296297 public static func format(_ bytes: Int64) -> String {298 let gb = Double(bytes) / 1_073_741_824299 if gb >= 1 { return String(format: "%.1f GB", gb) }300 return String(format: "%.0f MB", Double(bytes) / 1_048_576)301 }302}303304// MARK: - Thermal305306public enum ThermalLevel: String, Sendable, Codable, Hashable, CaseIterable, Comparable {307 case nominal, fair, serious, critical308 public init(_ state: ProcessInfo.ThermalState) {309 switch state {310 case .nominal: self = .nominal311 case .fair: self = .fair312 case .serious: self = .serious313 case .critical: self = .critical314 @unknown default: self = .fair315 }316 }317 private var rank: Int { switch self { case .nominal: 0; case .fair: 1; case .serious: 2; case .critical: 3 } }318 public static func < (a: ThermalLevel, b: ThermalLevel) -> Bool { a.rank < b.rank }319 public static func current() -> ThermalLevel { ThermalLevel(ProcessInfo.processInfo.thermalState) }320}321322/// What the app must do at a given thermal level.323public struct ThermalDecision: Sendable, Hashable {324 public var level: ThermalLevel325 public var allowsNewGeneration: Bool326 public var shouldUnload: Bool327 /// Multipliers applied to the requested limits (1 = unchanged).328 public var maxTokensFactor: Double329 public var contextFactor: Double330 /// Text for the UI; nil when there is nothing to say.331 public var warning: String?332333 /// Applies the reduction to request parameters. Never raises a limit.334 public func limit(maxTokens: Int?, context: Int?) -> (maxTokens: Int?, context: Int?) {335 guard maxTokensFactor < 1 || contextFactor < 1 else { return (maxTokens, context) }336 let tokens = Int(Double(maxTokens ?? ThermalGovernor.defaultMaxTokens) * maxTokensFactor)337 let ctx = context.map { max(1_024, Int(Double($0) * contextFactor)) }338 return (max(64, tokens), ctx)339 }340}341342/// Thermal policy: serious → shorter answers and smaller context, with a warning; critical → no new generation, unload. Pure.343public struct ThermalGovernor: Sendable {344 public static let defaultMaxTokens = 1_024345 public var seriousMaxTokensFactor = 0.5346 public var seriousContextFactor = 0.5347 public init() {}348349 public func decision(for level: ThermalLevel) -> ThermalDecision {350 switch level {351 case .nominal, .fair:352 ThermalDecision(level: level, allowsNewGeneration: true, shouldUnload: false, maxTokensFactor: 1, contextFactor: 1, warning: nil)353 case .serious:354 ThermalDecision(level: level, allowsNewGeneration: true, shouldUnload: false, maxTokensFactor: seriousMaxTokensFactor, contextFactor: seriousContextFactor,355 warning: "The device is hot. Answers are shortened and the context is reduced until it cools down.")356 case .critical:357 ThermalDecision(level: level, allowsNewGeneration: false, shouldUnload: true, maxTokensFactor: 0, contextFactor: 0,358 warning: "The device is too hot to run a model. The model was unloaded; try again when it has cooled down.")359 }360 }361362 /// Stream of thermal levels: the current one first, then every change reported by the system.363 public static func levels() -> AsyncStream<ThermalLevel> {364 AsyncStream(bufferingPolicy: .bufferingNewest(1)) { continuation in365 continuation.yield(.current())366 let box = ObserverBox(NotificationCenter.default.addObserver(forName: ProcessInfo.thermalStateDidChangeNotification, object: nil, queue: nil) { _ in367 continuation.yield(.current())368 })369 continuation.onTermination = { _ in NotificationCenter.default.removeObserver(box.token) }370 }371 }372}373374final class ObserverBox: @unchecked Sendable {375 let token: NSObjectProtocol376 init(_ token: NSObjectProtocol) { self.token = token }377}378