SPB Git

Repository Public MIT

ChatCompletions.swift 24.6 KB 434 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraProviders/Sources/ChatCompletions.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import AstraCore89import Foundation9091// The Chat Completions wire format (`POST …/chat/completions`, `data:` chunks ending with `[DONE]`),92// shared by OpenRouter, user-configured OpenAI-compatible endpoints and the keyed providers built on `KeyedChatCompletions`93// (Mistral, DeepSeek, Moonshot, Cerebras, DeepInfra).9495struct ChatCompletionsDialect: Sendable {96    enum Reasoning: Sendable {97        /// OpenRouter: `"reasoning": {"effort": …}`.98        case object99        /// Common OpenAI-compatible spelling: `"reasoning_effort": "low" | "medium" | "high"`.100        case effortField101        /// `reasoning_effort` where the provider also documents `"none"` to switch reasoning off (Mistral, Cerebras, DeepInfra).102        case effortFieldWithNone103        /// The encoder writes nothing; the dialect's `prepare` hook sets the provider's own fields (DeepSeek, Moonshot `thinking`).104        case managed105    }106    /// What a tool-calling turn must carry back in the next request, kept in `ToolCall.providerState` of the turn's first call.107    enum Replay: Sendable, Equatable {108        case none109        /// The turn's reasoning text, returned and replayed in the same assistant-message field110        /// (`reasoning_content` on DeepSeek and Moonshot, `reasoning` on Cerebras).111        case reasoningField(format: String, field: String)112        /// Mistral: the turn's `thinking` chunks, replayed as a typed chunk at the head of the assistant `content`.113        case thinkingChunks(format: String)114        /// OpenRouter: the `reasoning_details` array, replayed unmodified and in order.115        case reasoningDetails(format: String)116    }117    var reasoning: Reasoning118    var replay = Replay.none119    /// `stream_options: {"include_usage": true}`; OpenRouter always sends usage and ignores the option.120    var requestsUsage: Bool121    var includesRouting: Bool122    /// Name of the output cap: `max_tokens`, or `max_completion_tokens` where the provider documents that spelling.123    var maxTokensKey = "max_tokens"124    /// Name of the sampling seed (`random_seed` on Mistral, which rejects `seed`); nil when the provider has none.125    var seedKey: String? = "seed"126    /// Last edit of the encoded body: provider-specific fields that the shared encoder does not know.127    var prepare: (@Sendable (GenerationRequest, inout [String: JSONValue]) -> Void)?128    /// Provider-specific parameter fallback, tried before the generic one. Returns true when the body was changed and is worth resending.129    var adjust: (@Sendable (ProviderHTTP.Failure, inout [String: JSONValue]) -> Bool)?130    /// Rewrites a failure into the status class it really is (a provider that answers 400 for an unknown model, for instance).131    var normalize: @Sendable (ProviderHTTP.Failure) -> ProviderHTTP.Failure = { $0 }132}133134enum ChatCompletionsEncoder {135    static func body(_ request: GenerationRequest, dialect: ChatCompletionsDialect) -> [String: JSONValue] {136        var messages: [JSONValue] = []137        if let system = request.system?.trimmingCharacters(in: .whitespacesAndNewlines), !system.isEmpty {138            messages.append(["role": "system", "content": .string(system)])139        }140        for message in request.messages { messages.append(wire(message, replay: dialect.replay)) }141        var body: [String: JSONValue] = ["model": .string(request.model.modelID), "messages": .array(messages), "stream": true]142        if dialect.requestsUsage { body["stream_options"] = ["include_usage": true] }143        if !request.tools.isEmpty {144            body["tools"] = .array(request.tools.map { tool in145                ["type": "function", "function": ["name": .string(tool.name), "description": .string(tool.summary), "parameters": tool.parameters]]146            })147        }148        let p = request.parameters149        if let v = p.maxOutputTokens { body[dialect.maxTokensKey] = .number(Double(v)) }150        if let v = p.temperature { body["temperature"] = .number(v) }151        if let v = p.topP { body["top_p"] = .number(v) }152        if let v = p.seed, let key = dialect.seedKey { body[key] = .number(Double(v)) }153        if !p.stop.isEmpty { body["stop"] = .array(p.stop.prefix(4).map(JSONValue.string)) }154        if let effort = p.reasoning {155            switch dialect.reasoning {156            case .object: body["reasoning"] = ["effort": .string(effort == .off ? "none" : effort.rawValue)]157            case .effortField: if effort != .off { body["reasoning_effort"] = .string(effort.rawValue) }158            case .effortFieldWithNone: body["reasoning_effort"] = .string(effort == .off ? "none" : effort.rawValue)159            case .managed: break160            }161        }162        if dialect.includesRouting, let routing = request.routing, !routing.isDefault {163            var provider: [String: JSONValue] = [:]164            if !routing.providerOrder.isEmpty { provider["order"] = .array(routing.providerOrder.map(JSONValue.string)) }165            if !routing.allowFallbacks { provider["allow_fallbacks"] = false }166            if routing.denyDataCollection { provider["data_collection"] = "deny" }167            if !provider.isEmpty { body["provider"] = .object(provider) }168            let fallbacks = routing.fallbackModels.filter { $0 != request.model.modelID }169            if !fallbacks.isEmpty { body["models"] = .array(fallbacks.map(JSONValue.string)) }170        }171        dialect.prepare?(request, &body)172        return body173    }174175    static func wire(_ message: PromptMessage, replay: ChatCompletionsDialect.Replay = .none) -> JSONValue {176        switch message.role {177        case .user:178            if message.images.isEmpty { return ["role": "user", "content": .string(message.text)] }179            var parts: [JSONValue] = []180            if !message.text.isEmpty { parts.append(["type": "text", "text": .string(message.text)]) }181            parts += message.images.map { ["type": "image_url", "image_url": ["url": .string(dataURL($0))]] }182            return ["role": "user", "content": .array(parts)]183        case .assistant:184            var out: [String: JSONValue] = ["role": "assistant", "content": message.text.isEmpty && !message.toolCalls.isEmpty ? .null : .string(message.text)]185            if !message.toolCalls.isEmpty {186                out["tool_calls"] = .array(message.toolCalls.map { call in187                    ["id": .string(call.id), "type": "function", "function": ["name": .string(call.name), "arguments": .string(ToolArguments.wireText(call.arguments))]]188                })189                // What the provider asked to see again with this turn. A call without state (an older conversation,190                // another provider's turn) adds nothing and the message keeps its plain shape.191                switch replay {192                case .none: break193                case .reasoningField(let format, let field):194                    if let text = ProviderState.payload(message.toolCalls, format: format)?.stringValue { out[field] = .string(text) }195                case .thinkingChunks(let format):196                    if let text = ProviderState.payload(message.toolCalls, format: format)?.stringValue {197                        var chunks: [JSONValue] = [["type": "thinking", "thinking": [["type": "text", "text": .string(text)]]]]198                        if !message.text.isEmpty { chunks.append(["type": "text", "text": .string(message.text)]) }199                        out["content"] = .array(chunks)200                    }201                case .reasoningDetails(let format):202                    if let details = ProviderState.payload(message.toolCalls, format: format), details.arrayValue != nil { out["reasoning_details"] = details }203                }204            }205            return .object(out)206        case .tool:207            let text = message.toolIsError ? "Error: " + message.text : message.text208            return ["role": "tool", "tool_call_id": .string(message.toolCallID ?? ""), "content": .string(text)]209        }210    }211212    /// Edits the body after a parameter rejection by a compatible server. Returns false when nothing can be routed around.213    static func adjust(_ failure: ProviderHTTP.Failure, _ body: inout [String: JSONValue]) -> Bool {214        let message = (failure.body.message ?? "").lowercased()215        // Newer OpenAI-style servers want `max_completion_tokens` and say so.216        if failure.status == 400, message.contains("max_completion_tokens"), let value = body["max_tokens"] {217            body.removeValue(forKey: "max_tokens")218            body["max_completion_tokens"] = value219            return true220        }221        var candidates: [UnsupportedParameter.Candidate] = []222        for key in ["stream_options", "reasoning_effort", "reasoning", "thinking", "seed", "random_seed", "top_p", "temperature", "stop"] where body[key] != nil {223            candidates.append(.init(key, key == "stream_options" ? ["include_usage"] : []))224        }225        guard let hit = UnsupportedParameter.detect(failure, candidates: candidates) else { return false }226        body.removeValue(forKey: hit)227        return true228    }229}230231/// Turns chat-completion chunks into `StreamEvent`s. One instance per response.232struct ChatCompletionsStreamDecoder {233    private struct Call { var id: String; var name: String; var arguments = ""; var started = false }234235    let modelID: String236    let replay: ChatCompletionsDialect.Replay237    private(set) var started = false238    private(set) var finished = false239    private(set) var sawEvents = false240    private var calls: [Int: Call] = [:]241    private var finishReason: String?242    private var emittedCalls = false243    private var responseID: String?244    /// Reasoning of this turn, kept only to be handed back with its tool calls.245    private var replayText = ""246    private var replayDetails: [JSONValue] = []247248    init(modelID: String, replay: ChatCompletionsDialect.Replay = .none) { self.modelID = modelID; self.replay = replay }249250    /// The state the turn's first tool call carries, in the dialect's format.251    private func turnState() -> JSONValue? {252        switch replay {253        case .none: return nil254        case .reasoningField(let format, _), .thinkingChunks(let format): return ProviderState.make(format, .string(replayText))255        case .reasoningDetails(let format): return ProviderState.make(format, .array(replayDetails))256        }257    }258259    /// Streamed `reasoning_details` chunks of one entry (same `index` and `type`) are joined: text, summary and data are260    /// concatenated in arrival order, every other field keeps the last value sent.261    private mutating func merge(details: [JSONValue]) {262        for chunk in details {263            guard case .object(let fields) = chunk else { continue }264            let position = replayDetails.lastIndex { $0["index"] == fields["index"] && $0["type"] == fields["type"] && fields["index"] != nil }265            guard let position, case .object(var merged) = replayDetails[position] else { replayDetails.append(chunk); continue }266            for (key, value) in fields {267                if ["text", "summary", "data"].contains(key), let old = merged[key]?.stringValue, let new = value.stringValue { merged[key] = .string(old + new) }268                else if !value.isNull || merged[key] == nil { merged[key] = value }269            }270            replayDetails[position] = .object(merged)271        }272    }273274    /// Feeds the `data:` payload of one SSE event.275    mutating func consume(_ data: String) throws -> [StreamEvent] {276        let trimmed = data.trimmingCharacters(in: .whitespaces)277        if trimmed.isEmpty { return [] }278        sawEvents = true279        if trimmed == "[DONE]" { return finish() }280        guard let json = JSONValue.parse(trimmed) else { throw AstraError.decoding("a stream chunk was not valid JSON") }281        return try consume(json)282    }283284    mutating func consume(_ json: JSONValue) throws -> [StreamEvent] {285        guard !finished else { return [] }286        if let error = json["error"], !error.isNull { throw Self.streamError(error, modelID: modelID) }287        var out: [StreamEvent] = []288        if !started {289            started = true290            responseID = json["id"]?.stringValue291            out.append(.messageStarted(MessageStart(responseID: responseID, reportedModel: json["model"]?.stringValue, servedBy: json["provider"]?.stringValue)))292        }293        for choice in json["choices"]?.arrayValue ?? [] where (choice["index"]?.intValue ?? 0) == 0 {294            // Streaming servers use `delta`; a server that ignored `stream` answers with a whole `message`.295            let delta = choice["delta"] ?? choice["message"]296            if let text = delta?["reasoning"]?.stringValue ?? delta?["reasoning_content"]?.stringValue, !text.isEmpty { out.append(.reasoningSummaryDelta(text)) }297            if let text = delta?["content"]?.stringValue, !text.isEmpty { out.append(.textDelta(text)) }298            switch replay {299            case .reasoningField(_, let field): if let text = delta?[field]?.stringValue { replayText += text }300            case .reasoningDetails: if let details = delta?["reasoning_details"]?.arrayValue { merge(details: details) }301            case .none, .thinkingChunks: break302            }303            // Mistral reasoning models send `content` as typed chunks: `{"type": "thinking", "thinking": [{"type": "text", "text"}]}`304            // carries the reasoning trace, `{"type": "text", "text"}` the answer. Other chunk types (references, files) are not text.305            for chunk in delta?["content"]?.arrayValue ?? [] {306                switch chunk["type"]?.stringValue {307                case "thinking":308                    let text = chunk["thinking"]?.arrayValue?.compactMap { $0["text"]?.stringValue }.joined() ?? chunk["thinking"]?.stringValue ?? ""309                    if !text.isEmpty { out.append(.reasoningSummaryDelta(text)) }310                    if case .thinkingChunks = replay { replayText += text }311                case "text":312                    if let text = chunk["text"]?.stringValue, !text.isEmpty { out.append(.textDelta(text)) }313                default: break314                }315            }316            if let text = delta?["refusal"]?.stringValue, !text.isEmpty { out.append(.textDelta(text)) }317            for (position, fragment) in (delta?["tool_calls"]?.arrayValue ?? []).enumerated() {318                let index = fragment["index"]?.intValue ?? position319                var call = calls[index] ?? Call(id: "", name: "")320                if let id = fragment["id"]?.stringValue, !id.isEmpty, call.id.isEmpty { call.id = id }321                if let name = fragment["function"]?["name"]?.stringValue, !name.isEmpty, call.name.isEmpty { call.name = name }322                let arguments = fragment["function"]?["arguments"]?.stringValue ?? ""323                if !call.started, !call.name.isEmpty {324                    // Some servers never send an id: synthesise one that is stable within the response.325                    if call.id.isEmpty { call.id = "call_\(responseID ?? "astra")_\(index)" }326                    call.started = true327                    emittedCalls = true328                    out.append(.toolCallStarted(id: call.id, name: call.name))329                    if !call.arguments.isEmpty { out.append(.toolCallDelta(id: call.id, argumentsFragment: call.arguments)) }330                }331                call.arguments += arguments332                if call.started, !arguments.isEmpty { out.append(.toolCallDelta(id: call.id, argumentsFragment: arguments)) }333                calls[index] = call334            }335            if let reason = choice["finish_reason"]?.stringValue, !reason.isEmpty {336                if reason == "error" { throw AstraError.server(status: 502, message: "The provider ended the answer with an error.") }337                finishReason = reason338            }339            // Moonshot reports usage inside the finishing choice rather than at the top level.340            if let usage = Self.usage(choice["usage"]) { out.append(.usage(usage)) }341        }342        if let usage = Self.usage(json["usage"]) { out.append(.usage(usage)) }343        return out344    }345346    /// Ends the message: closes open tool calls and emits the single terminal event. Safe to call more than once.347    mutating func finish() -> [StreamEvent] {348        guard !finished else { return [] }349        finished = true350        var out: [StreamEvent] = []351        if !started { started = true; out.append(.messageStarted(MessageStart())) }352        var state = turnState()353        for index in calls.keys.sorted() {354            guard let call = calls[index], call.started else { continue }355            out.append(.toolCallCompleted(ToolCall(id: call.id, name: call.name, arguments: ToolArguments.parse(call.arguments), providerState: state)))356            state = nil     // turn-level state rides on the first call only357        }358        out.append(.messageCompleted(stopReason()))359        return out360    }361362    /// True once the provider has said how the message ended, even if `[DONE]` never arrives.363    var hasFinishReason: Bool { finishReason != nil }364365    private func stopReason() -> StopReason {366        switch finishReason {367        case "tool_calls", "function_call": return .toolCalls368        case "stop", "end_turn", "eos": return emittedCalls ? .toolCalls : .endOfTurn369        case "length", "max_tokens": return .maxTokens370        case "content_filter": return .contentFilter371        case "stop_sequence": return .stopSequence372        case nil: return emittedCalls ? .toolCalls : .unknown373        default: return .unknown374        }375    }376377    /// `prompt_tokens` is the whole prompt; cached and cache-write counts are subsets. `cost` is OpenRouter's own figure in US dollars.378    /// Cached input is `prompt_tokens_details.cached_tokens`; DeepSeek also sends `prompt_cache_hit_tokens`, Moonshot a flat `cached_tokens`.379    static func usage(_ json: JSONValue?) -> TokenUsage? {380        guard let json, json.objectValue != nil else { return nil }381        return TokenUsage(inputTokens: json["prompt_tokens"]?.intValue, outputTokens: json["completion_tokens"]?.intValue,382                          cachedInputTokens: json["prompt_tokens_details"]?["cached_tokens"]?.intValue ?? json["prompt_cache_hit_tokens"]?.intValue383                              ?? json["cached_tokens"]?.intValue,384                          cacheWriteTokens: json["prompt_tokens_details"]?["cache_write_tokens"]?.intValue,385                          reasoningTokens: json["completion_tokens_details"]?["reasoning_tokens"]?.intValue,386                          reportedCostUSD: json["cost"]?.doubleValue)387    }388389    /// An `error` object inside a 200 stream: its numeric `code` is an HTTP status (OpenRouter), otherwise classify by type.390    static func streamError(_ error: JSONValue, modelID: String) -> AstraError {391        let body = ProviderErrorBody.parse(.object(["error": error]))392        var status = error["code"]?.intValue ?? error["status"]?.intValue ?? 0393        if !(400..<600).contains(status) {394            let code = (body.code ?? "").lowercased()395            status = code.contains("rate_limit") ? 429 : code.contains("overloaded") || code.contains("unavailable") ? 503 : code.contains("context_length") || code.contains("invalid") ? 400 : 502396        }397        return mapHTTPError(status: status, code: body.code, message: body.message, retryAfter: nil, model: modelID)398    }399}400401/// Drives one chat-completions generation: request (with parameter fallback), chunk decoding, terminal-event guarantee.402func runChatCompletionsStream(http: ProviderHTTP, request: GenerationRequest, dialect: ChatCompletionsDialect, secrets: [String],403                              makeRequest: (JSONValue) throws -> URLRequest, continuation: EventContinuation) async throws {404    let modelID = request.model.modelID405    let body = ChatCompletionsEncoder.body(request, dialect: dialect)406    let (_, lines) = try await http.openAdjusting(body: body, modelID: modelID, secrets: secrets, makeRequest: makeRequest, normalize: dialect.normalize) { failure, body in407        if let adjust = dialect.adjust, adjust(failure, &body) { return true }408        return ChatCompletionsEncoder.adjust(failure, &body)409    }410    var parser = SSEParser()411    var decoder = ChatCompletionsStreamDecoder(modelID: modelID, replay: dialect.replay)412    var plain = ""   // a body that is not SSE at all (an error or a whole completion sent as JSON with status 200)413    for try await line in lines {414        if !decoder.sawEvents, !line.isEmpty, !line.hasPrefix("data:"), !line.hasPrefix(":"), !line.hasPrefix("event:"), !line.hasPrefix("id:"), !line.hasPrefix("retry:") {415            if plain.utf8.count < 4_000_000 { plain += line + "\n" }416            continue417        }418        guard let event = parser.feed(line) else { continue }419        for out in try decoder.consume(event.data) { continuation.yield(out) }420        if decoder.finished { return }421    }422    if let event = parser.finish() { for out in try decoder.consume(event.data) { continuation.yield(out) } }423    try Task.checkCancellation()424    if decoder.finished { return }425    if !decoder.sawEvents, let json = JSONValue.parse(plain) {426        for out in try decoder.consume(json) { continuation.yield(out) }427        for out in decoder.finish() { continuation.yield(out) }428        return429    }430    // Some servers close the connection after the final chunk without sending [DONE].431    guard decoder.hasFinishReason else { throw AstraError.network("The connection closed before the answer was complete.") }432    for out in decoder.finish() { continuation.yield(out) }433}434
Files main
Menu

Theme