SPB Git

Repository Public MIT

Arena.swift 12.5 KB 239 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX Astra9// File             : Packages/AstraPersistence/Sources/Arena.swift10// Module           : Packages11// Repository       : https://git.spboucher.ai/orix-astra12// Created          : 2026-11-2113// Last Modified    : 2026-11-2114//15// =============================================================================16// AUTHOR & PROFESSIONAL AFFILIATION17// =============================================================================18//19// Author           : Simon-Pierre Boucher20// Position         : Professor of Finance21// Institution      : Université du Québec en Outaouais (UQO)22// Department       : Département des sciences administratives23// Organization     : SP Boucher24//25// Website          : https://spboucher.ai26// Git              : https://git.spboucher.ai27//28// =============================================================================29// OFFICIAL CONTACT DIRECTORY30// =============================================================================31//32// General          : info@spboucher.ai33// Professional     : contact@spboucher.ai34// Direct           : simon@spboucher.ai35// Research         : research@spboucher.ai36// Development      : dev@spboucher.ai37// Support          : support@spboucher.ai38// Security/Admin   : admin@spboucher.ai39// Notifications    : notifications@spboucher.ai40//41// Preferred Contact: support@spboucher.ai42//43// =============================================================================44// COPYRIGHT & INTELLECTUAL PROPERTY45// =============================================================================46//47// Copyright (c) 202648// Simon-Pierre Boucher49//50// All rights reserved where applicable.51//52// This file is governed by the applicable project license,53// copyright notices, institutional agreements and relevant law.54//55// Unless expressly authorized by the applicable license or56// rights holder, copying, redistribution, sublicensing,57// publication and commercial exploitation are not permitted.58//59// Existing third-party rights and notices remain unaffected.60//61// =============================================================================62// CONFIDENTIALITY & SECURITY63// =============================================================================64//65// Classification   : Public66// Distribution     : Public repository; use is governed by the project license67// Security Contact : admin@spboucher.ai68//69// Access to this source does not independently grant permission70// to redistribute or disclose confidential materials.71//72// Do not embed credentials, private keys, access tokens,73// certificates or confidential personal information.74//75// =============================================================================76// LICENSE & MAINTENANCE77// =============================================================================78//79// License          : Proprietary, all rights reserved (see license file)80// License File     : LICENSE81// Maintainer       : Simon-Pierre Boucher82// Technical Contact: dev@spboucher.ai83//84// =============================================================================85// END OF SP BOUCHER SOURCE HEADER86// =============================================================================8788import Foundation89import GRDB90import AstraCore9192public extension AstraStore {93    /// Creates a session with one pending entry per competitor. In a blind session the display order is shuffled.94    @discardableResult95    func createArenaSession(title: String = "", prompt: String, system: String = "", parameters: GenerationParameters = .default,96                            blind: Bool = false, templateID: String? = nil, competitors: [ModelRef],97                            at date: Date = Date()) async throws -> (session: ArenaSessionRecord, entries: [ArenaEntryRecord]) {98        let session = ArenaSessionRecord(id: AstraID.new(), title: title, prompt: prompt, system: system, parameters: parameters, blind: blind,99                                         createdAt: date, templateID: templateID)100        let order = blind ? Array(competitors.indices).shuffled() : Array(competitors.indices)101        let entries = competitors.enumerated().map { index, model in102            ArenaEntryRecord(id: AstraID.new(), sessionID: session.id, ordinal: index, model: model, reportedModel: nil, servedBy: nil, text: "",103                             status: .pending, usage: nil, timeToFirstToken: nil, totalTime: nil, tokensPerSecond: nil, cost: nil, error: nil,104                             stopReason: nil, isWinner: false, revealOrder: order[index], createdAt: date)105        }106        try await write { db in107            try session.insert(db)108            for entry in entries { try entry.insert(db) }109        }110        return (session, entries)111    }112113    /// Most recent first.114    func arenaSessions(limit: Int? = nil) async throws -> [ArenaSessionRecord] {115        try await read { db in try ArenaSessionRecord.fetchAll(db, sql: "SELECT * FROM arenaSession ORDER BY createdAt DESC, id LIMIT ?", arguments: [limit ?? -1]) }116    }117118    func arenaSession(id: String) async throws -> ArenaSessionRecord? { try await read { db in try ArenaSessionRecord.find(db, id: id) } }119120    /// Entries in display order (`revealOrder`).121    func arenaEntries(sessionID: String) async throws -> [ArenaEntryRecord] { try await read { db in try Self.arenaEntries(db, sessionID) } }122123    /// Saves an entry's progress or result (text, usage, timings, cost, error).124    func updateArenaEntry(_ entry: ArenaEntryRecord) async throws { try await write { db in try entry.updateByID(db) } }125126    /// Saves a finished entry and writes its usage ledger row in the same transaction.127    func finishArenaEntry(_ entry: ArenaEntryRecord, recordUsage: Bool = true, at date: Date = Date()) async throws {128        try await write { db in129            try entry.updateByID(db)130            if recordUsage, entry.usage != nil || entry.cost != nil {131                try UsageRecord(id: AstraID.new(), date: date, model: entry.model, usage: entry.usage, cost: entry.cost, origin: .arena,132                                originID: entry.sessionID).insert(db)133            }134        }135    }136137    /// Marks one entry as the winner (nil clears the choice).138    func setArenaWinner(sessionID: String, entryID: String?) async throws {139        try await write { db in140            try db.execute(sql: "UPDATE arenaEntry SET isWinner = (id IS ?) WHERE sessionID = ?", arguments: [entryID, sessionID])141        }142    }143144    func renameArenaSession(id: String, title: String) async throws {145        try await write { db in try db.execute(sql: "UPDATE arenaSession SET title = ? WHERE id = ?", arguments: [title, id]) }146    }147148    func deleteArenaSession(id: String) async throws { try await write { db in _ = try ArenaSessionRecord.remove(db, id: id) } }149150    /// Exports a session. `anonymized` replaces model names by "Model A", "Model B"... in display order (for sharing a blind test).151    func exportArenaSession(id: String, format: ExportFormat, anonymized: Bool = false) async throws -> String {152        let (session, entries) = try await read { db in153            guard let session = try ArenaSessionRecord.find(db, id: id) else { throw AstraStoreError.notFound("This arena session") }154            return (session, try Self.arenaEntries(db, id))155        }156        return try Self.render(arena: session, entries: entries, format: format, anonymized: anonymized)157    }158}159160/// Shape of the JSON export of an arena session.161public struct ArenaExport: Sendable, Codable, Hashable {162    public struct Entry: Sendable, Codable, Hashable {163        public var label: String164        public var provider: String?165        public var model: String?166        public var reportedModel: String?167        public var servedBy: String?168        public var isWinner: Bool169        public var status: ArenaEntryStatus170        public var stopReason: StopReason?171        public var usage: TokenUsage?172        public var timeToFirstToken: TimeInterval?173        public var totalTime: TimeInterval?174        public var tokensPerSecond: Double?175        public var cost: CostEstimate?176        public var error: String?177        public var text: String178    }179    public var format: String180    public var exportedAt: Date181    public var id: String182    public var title: String183    public var createdAt: Date184    public var blind: Bool185    public var anonymized: Bool186    public var system: String187    public var prompt: String188    public var parameters: GenerationParameters189    public var entries: [Entry]190}191192extension AstraStore {193    static func arenaEntries(_ db: Database, _ sessionID: String) throws -> [ArenaEntryRecord] {194        try ArenaEntryRecord.fetchAll(db, sql: "SELECT * FROM arenaEntry WHERE sessionID = ? ORDER BY revealOrder, ordinal, id", arguments: [sessionID])195    }196197    static func render(arena session: ArenaSessionRecord, entries: [ArenaEntryRecord], format: ExportFormat, anonymized: Bool,198                       exportedAt: Date = Date()) throws -> String {199        let rows = entries.map { e in200            ArenaExport.Entry(label: anonymized ? e.blindLabel : e.model.modelID, provider: anonymized ? nil : e.model.provider.rawValue,201                              model: anonymized ? nil : e.model.modelID, reportedModel: anonymized ? nil : e.reportedModel,202                              servedBy: anonymized ? nil : e.servedBy, isWinner: e.isWinner, status: e.status, stopReason: e.stopReason,203                              usage: e.usage, timeToFirstToken: e.timeToFirstToken, totalTime: e.totalTime, tokensPerSecond: e.tokensPerSecond,204                              cost: e.cost, error: e.error.map { Redactor.redact($0.message) }, text: e.text)205        }206        switch format {207        case .json:208            let export = ArenaExport(format: "orix-astra-arena/1", exportedAt: exportedAt, id: session.id, title: session.title,209                                     createdAt: session.createdAt, blind: session.blind, anonymized: anonymized, system: session.system,210                                     prompt: session.prompt, parameters: session.parameters, entries: rows)211            return String(decoding: try ExportCoding.encoder.encode(export), as: UTF8.self)212        case .csv:213            let header = ["label", "provider", "model", "reported_model", "served_by", "winner", "status", "stop_reason", "input_tokens",214                          "output_tokens", "ttft_s", "total_s", "tokens_per_second", "cost_usd", "cost_basis", "error", "prompt", "response"]215            return CSVWriter.document(header: header, rows: rows.map { r in216                [r.label, r.provider ?? "", r.model ?? "", r.reportedModel ?? "", r.servedBy ?? "", r.isWinner ? "yes" : "no", r.status.rawValue,217                 r.stopReason?.rawValue ?? "", CSVWriter.number(r.usage?.inputTokens), CSVWriter.number(r.usage?.outputTokens),218                 CSVWriter.number(r.timeToFirstToken), CSVWriter.number(r.totalTime), CSVWriter.number(r.tokensPerSecond),219                 CSVWriter.number(r.cost?.amountUSD), r.cost?.basis.rawValue ?? "", r.error ?? "", session.prompt, r.text]220            })221        case .markdown:222            var out = "# \(session.title.isEmpty ? "Arena session" : session.title)\n\n"223            out += "- Date: \(ExportCoding.iso(session.createdAt))\n- Blind: \(session.blind ? "yes" : "no")\n\n"224            if !session.system.isEmpty { out += "## System\n\n\(Markdown.fenced(session.system))\n\n" }225            out += "## Prompt\n\n\(Markdown.fenced(session.prompt))\n\n## Results\n\n"226            out += Markdown.table(header: ["Model", "Winner", "Output tokens", "First token (s)", "Total (s)", "Tokens/s", "Cost"], rows: rows.map { r in227                [r.label, r.isWinner ? "yes" : "", CSVWriter.number(r.usage?.outputTokens), r.timeToFirstToken.map { String(format: "%.2f", $0) } ?? "",228                 r.totalTime.map { String(format: "%.2f", $0) } ?? "", r.tokensPerSecond.map { String(format: "%.1f", $0) } ?? "", r.cost?.label ?? ""]229            })230            for r in rows {231                out += "\n## \(r.label)\(r.isWinner ? " (winner)" : "")\n\n"232                if let error = r.error { out += "> Error: \(error.replacingOccurrences(of: "\n", with: " "))\n\n" }233                out += r.text.isEmpty ? "_No answer._\n" : r.text + (r.text.hasSuffix("\n") ? "" : "\n")234            }235            return out236        }237    }238}239
Files main
Menu

Theme