SPB Git

Repository Public MIT

Provider.swift 14.7 KB 293 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraCore/Sources/Provider.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2214//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation8990/// Result of a provider connection check: proves the key works without generating anything.91public struct ConnectionStatus: Sendable, Hashable, Codable {92    public var ok: Bool93    public var detail: String94    public var modelCount: Int?95    /// Remaining credit in US dollars when the provider reports it (OpenRouter).96    public var remainingCreditUSD: Double?97    public var checkedAt: Date98    public init(ok: Bool, detail: String, modelCount: Int? = nil, remainingCreditUSD: Double? = nil, checkedAt: Date = Date()) {99        self.ok = ok; self.detail = detail; self.modelCount = modelCount; self.remainingCreditUSD = remainingCreditUSD; self.checkedAt = checkedAt100    }101}102103/// One inference backend, cloud or local. Adapters translate to and from their transport; nothing else knows the wire format.104public protocol ChatProvider: Sendable {105    var kind: ProviderKind { get }106    /// Models this provider offers right now, with metadata that states its own source.107    func listModels() async throws -> [ModelInfo]108    /// Verifies credentials and reachability with the cheapest call the provider has.109    func verify() async throws -> ConnectionStatus110    /// Streams one answer. Cancelling the consuming task cancels the request.111    func stream(_ request: GenerationRequest) -> AsyncThrowingStream<StreamEvent, Error>112}113114/// One sanitized record per provider request, for Lab diagnostics. Never contains headers, keys, prompts or answers.115public struct RequestDiagnostic: Sendable, Hashable, Codable, Identifiable {116    public var id: UUID117    public var provider: ProviderKind118    public var modelID: String119    public var startedAt: Date120    public var httpStatus: Int?121    public var requestID: String?122    public var timeToHeaders: TimeInterval?123    public var timeToFirstToken: TimeInterval?124    public var totalTime: TimeInterval?125    public var retryCount: Int126    public var usage: TokenUsage?127    public var stopReason: StopReason?128    public var errorTitle: String?129    public init(id: UUID = UUID(), provider: ProviderKind, modelID: String, startedAt: Date = Date(), httpStatus: Int? = nil, requestID: String? = nil,130                timeToHeaders: TimeInterval? = nil, timeToFirstToken: TimeInterval? = nil, totalTime: TimeInterval? = nil, retryCount: Int = 0,131                usage: TokenUsage? = nil, stopReason: StopReason? = nil, errorTitle: String? = nil) {132        self.id = id; self.provider = provider; self.modelID = modelID; self.startedAt = startedAt; self.httpStatus = httpStatus133        self.requestID = requestID; self.timeToHeaders = timeToHeaders; self.timeToFirstToken = timeToFirstToken; self.totalTime = totalTime134        self.retryCount = retryCount; self.usage = usage; self.stopReason = stopReason; self.errorTitle = errorTitle135    }136}137138/// Receives transport-level facts from adapters (status, request id, header latency). Implemented by the app's diagnostics store.139public protocol DiagnosticsSink: Sendable {140    func httpResponse(provider: ProviderKind, modelID: String, status: Int, requestID: String?, timeToHeaders: TimeInterval, retryCount: Int) async141}142143// MARK: - Stream validation and measurement144145/// Checks the order of events coming out of an adapter and turns a malformed stream into one clean terminal error.146/// Rules: one `messageStarted` first; tool deltas only for started calls; nothing after a terminal event; exactly one terminal event.147public struct StreamValidator: Sendable {148    public private(set) var started = false149    public private(set) var finished = false150    private var openCalls: Set<String> = []151    private var completedCalls: Set<String> = []152    public init() {}153154    /// Returns the events to forward for `event` (possibly a synthesized `messageStarted` before it), or throws on a violation.155    public mutating func accept(_ event: StreamEvent) throws -> [StreamEvent] {156        guard !finished else { throw AstraError.protocolViolation("event after the end of the message") }157        var out: [StreamEvent] = []158        if case .messageStarted = event {159            guard !started else { return [] }   // a duplicated start is harmless: keep the first identity160            started = true161            return [event]162        }163        if !started { started = true; out.append(.messageStarted(MessageStart())) }164        switch event {165        case .toolCallStarted(let id, _):166            guard !openCalls.contains(id), !completedCalls.contains(id) else { throw AstraError.protocolViolation("tool call \(id) started twice") }167            openCalls.insert(id)168        case .toolCallDelta(let id, _):169            guard openCalls.contains(id) else { throw AstraError.protocolViolation("arguments for a tool call that was never started") }170        case .toolCallCompleted(let call):171            guard !completedCalls.contains(call.id) else { throw AstraError.protocolViolation("tool call \(call.id) completed twice") }172            openCalls.remove(call.id); completedCalls.insert(call.id)173        case .messageCompleted, .error:174            finished = true175        default: break176        }177        out.append(event)178        return out179    }180}181182/// Wall-clock measurements of one generation, taken by the orchestrator, never by the model.183public struct GenerationTiming: Sendable, Hashable, Codable {184    public var startedAt: Date185    public var timeToFirstToken: TimeInterval?186    public var totalTime: TimeInterval?187    public init(startedAt: Date = Date(), timeToFirstToken: TimeInterval? = nil, totalTime: TimeInterval? = nil) {188        self.startedAt = startedAt; self.timeToFirstToken = timeToFirstToken; self.totalTime = totalTime189    }190    /// Visible output tokens per second over the streaming phase (first visible token to the end).191    /// Nil unless it is a meaningful measurement: the token count was reported, hidden reasoning tokens are known and192    /// excluded (they are produced before the first visible token), and text actually streamed for at least half a second.193    /// A provider that delivers its whole answer in one burst therefore shows no speed rather than an absurd one.194    public func tokensPerSecond(outputTokens: Int?, reasoningTokens: Int? = nil) -> Double? {195        guard let outputTokens, let total = totalTime, let first = timeToFirstToken else { return nil }196        let visible = outputTokens - (reasoningTokens ?? 0)197        let phase = total - first198        guard visible > 1, phase >= 0.5 else { return nil }199        return Double(visible) / phase200    }201}202203/// The complete, provider-independent result of one generation.204public struct GenerationOutcome: Sendable, Hashable {205    public var start = MessageStart()206    public var text = ""207    public var reasoningSummary = ""208    public var toolCalls: [ToolCall] = []209    public var usage: TokenUsage?210    public var stopReason: StopReason = .unknown211    public var error: AstraError?212    public var timing = GenerationTiming()213    public init() {}214215    /// Folds one validated event into the outcome.216    public mutating func apply(_ event: StreamEvent, now: Date = Date()) {217        switch event {218        case .messageStarted(let s): start = s219        case .textDelta(let t):220            if timing.timeToFirstToken == nil { timing.timeToFirstToken = now.timeIntervalSince(timing.startedAt) }221            text += t222        case .reasoningSummaryDelta(let t):223            if timing.timeToFirstToken == nil { timing.timeToFirstToken = now.timeIntervalSince(timing.startedAt) }224            reasoningSummary += t225        case .toolCallStarted, .toolCallDelta: break226        case .toolCallCompleted(let call): toolCalls.append(call)227        case .usage(let u): usage = usage?.merged(with: u) ?? u228        case .messageCompleted(let reason): stopReason = reason; timing.totalTime = now.timeIntervalSince(timing.startedAt)229        case .error(let e): error = e; stopReason = e == .cancelled ? .cancelled : .error; timing.totalTime = now.timeIntervalSince(timing.startedAt)230        }231    }232}233234/// Routes requests to providers and guarantees a well-formed event stream: validated order, exactly one terminal event,235/// a safe retry only when nothing has been received yet, and cancellation that always ends with `.error(.cancelled)`.236public struct InferenceOrchestrator: Sendable {237    public typealias Resolver = @Sendable (ModelRef) async throws -> any ChatProvider238    private let resolve: Resolver239    public var maxRetries: Int240    public var sleep: @Sendable (TimeInterval) async throws -> Void241242    public init(maxRetries: Int = 1, sleep: @escaping @Sendable (TimeInterval) async throws -> Void = { try await Task.sleep(nanoseconds: UInt64($0 * 1_000_000_000)) },243                resolve: @escaping Resolver) {244        self.resolve = resolve; self.maxRetries = maxRetries; self.sleep = sleep245    }246247    /// Never throws: failures arrive as a terminal `.error` event, so partial text already delivered stays valid.248    public func stream(_ request: GenerationRequest) -> AsyncStream<StreamEvent> {249        // Bounded buffering applies backpressure hints without ever dropping an event.250        AsyncStream(bufferingPolicy: .unbounded) { continuation in251            let task = Task {252                var attempt = 0253                while true {254                    var validator = StreamValidator()255                    var receivedContent = false256                    do {257                        try Task.checkCancellation()258                        let provider = try await resolve(request.model)259                        for try await event in provider.stream(request) {260                            try Task.checkCancellation()261                            for out in try validator.accept(event) {262                                switch out {263                                case .messageStarted: break264                                default: receivedContent = true265                                }266                                continuation.yield(out)267                            }268                            if validator.finished { break }269                        }270                        if !validator.finished {271                            if Task.isCancelled { continuation.yield(.error(.cancelled)) }272                            else { continuation.yield(.error(.protocolViolation("the stream ended without completing the message"))) }273                        }274                        continuation.finish(); return275                    } catch {276                        let mapped = Task.isCancelled ? AstraError.cancelled : AstraError.wrap(error)277                        // Retrying is only safe when the provider has produced nothing: no partial answer, no tool call, no billed output shown.278                        if mapped.isTransient, !receivedContent, attempt < maxRetries, !Task.isCancelled {279                            attempt += 1280                            var delay = min(8, pow(2, Double(attempt - 1)))281                            if case .rateLimited(let after, _) = mapped, let after { delay = after }282                            if delay <= 20, (try? await sleep(delay)) != nil { continue }283                        }284                        if !validator.finished { continuation.yield(.error(mapped)) }285                        continuation.finish(); return286                    }287                }288            }289            continuation.onTermination = { _ in task.cancel() }290        }291    }292}293
Files main
Menu

Theme