SPB Git

Repository Public MIT

LocalRuntimeMonitor.swift 6.8 KB 159 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraLocal/Sources/LocalRuntimeMonitor.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation9091/// Figures measured during one local generation. A nil field was not measured; nothing here is estimated.92public struct LocalGenerationMetrics: Sendable, Hashable, Codable, Identifiable {93    public var id = UUID()94    public var modelID: String95    public var startedAt: Date96    /// Wall-clock seconds from the request to the first token of text or thinking.97    public var timeToFirstToken: TimeInterval?98    public var totalTime: TimeInterval99    /// Prompt tokens as the runtime counted them, cached ones included.100    public var promptTokens: Int?101    /// Prompt tokens served from the KV cache of an earlier turn.102    public var cachedPromptTokens: Int?103    public var completionTokens: Int?104    public var reasoningTokens: Int?105    public var promptTime: TimeInterval?106    public var generationTime: TimeInterval?107    /// Newly evaluated prompt tokens per second.108    public var promptTokensPerSecond: Double?109    /// Generated tokens per second over the decode phase.110    public var decodeTokensPerSecond: Double?111    /// Peak MLX allocator memory during this generation (weights included).112    public var peakMemoryBytes: Int64?113    public var activeMemoryBytes: Int64?114    public var stopReason: StopReason115    /// True for the first generation after the model was loaded.116    public var isColdStart = false117    /// True when the KV cache of an earlier turn was continued.118    public var reusedSession = false119    public var thermalBefore: ThermalLevel?120    public var thermalAfter: ThermalLevel?121122    public init(modelID: String, startedAt: Date = Date(), totalTime: TimeInterval = 0, stopReason: StopReason = .unknown) {123        self.modelID = modelID; self.startedAt = startedAt; self.totalTime = totalTime; self.stopReason = stopReason124    }125}126127/// Keeps the measured metrics of recent local generations and publishes each new one.128public final class LocalRuntimeMonitor: @unchecked Sendable {129    private let lock = NSLock()130    private var recent: [LocalGenerationMetrics] = []131    private var observers: [UUID: AsyncStream<LocalGenerationMetrics>.Continuation] = [:]132    private let capacity: Int133134    public init(capacity: Int = 50) { self.capacity = max(1, capacity) }135136    /// Metrics of the most recent generation, cancelled and failed ones included.137    public var lastMetrics: LocalGenerationMetrics? { lock.withLock { recent.last } }138    /// Oldest first.139    public var history: [LocalGenerationMetrics] { lock.withLock { recent } }140141    /// Every generation's metrics from now on.142    public func updates() -> AsyncStream<LocalGenerationMetrics> {143        let key = UUID()144        return AsyncStream(bufferingPolicy: .bufferingNewest(16)) { continuation in145            lock.withLock { observers[key] = continuation }146            continuation.onTermination = { [weak self] _ in self?.lock.withLock { self?.observers[key] = nil } }147        }148    }149150    public func record(_ metrics: LocalGenerationMetrics) {151        let targets = lock.withLock { () -> [AsyncStream<LocalGenerationMetrics>.Continuation] in152            recent.append(metrics)153            if recent.count > capacity { recent.removeFirst(recent.count - capacity) }154            return Array(observers.values)155        }156        for t in targets { t.yield(metrics) }157    }158}159
Files main
Menu

Theme