spb/zyquo-cloud Public MIT

1//2//  VerifyHarness.swift3//  Zyquo Cloud4//5//  Author: Simon-Pierre Boucher6//  Mail: contact@spboucher.ai7//8//  Phase 7 API verification: exercises the exact production provider clients9//  against live APIs with real keys. Run via `ZyquoCloud --verify` (wrapped by10//  the zyquo-verify target and `make verify`). Keys come from environment11//  variables (source .env.keys); they are never logged or persisted here.12//13//  Per provider: /models diff vs catalog, a minimal "Reply with exactly: OK"14//  completion on EVERY catalog chat model, a streaming test, and a vision test15//  where supported. Results: stdout table + docs/VERIFICATION.md.16//1718import Foundation1920enum VerifyHarness {21    // MARK: - Result model2223    struct TestResult {24        let provider: ProviderID25        let model: String26        let test: String27        let passed: Bool28        let latency: TimeInterval?29        let detail: String?30    }3132    static let environmentKeys: [ProviderID: String] = [33        .openai: "OPENAI_API_KEY",34        .anthropic: "ANTHROPIC_API_KEY",35        .xai: "XAI_API_KEY",36        .mistral: "MISTRAL_API_KEY",37        .gemini: "GEMINI_API_KEY",38        .qwen: "DASHSCOPE_API_KEY",39        .deepseek: "DEEPSEEK_API_KEY",40        .kimi: "MOONSHOT_API_KEY",41        .perplexity: "PERPLEXITY_API_KEY",42        .together: "TOGETHER_API_KEY",43        .deepinfra: "DEEPINFRA_API_KEY",44        .cerebras: "CEREBRAS_API_KEY",45    ]4647    /// Long-running or per-request-billed models excluded from the bulk "OK"48    /// sweep (each documented in docs/VERIFICATION.md).49    static let skipList: [String: String] = [50        "sonar-deep-research": "multi-minute agentic research runs; verified via docs only",51    ]5253    /// 64×64 red PNG for vision tests (several providers reject tiny images).54    static let redPixelPNG = Data(base64Encoded:55        "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PQQkAAAgAsetfWiP4FgYrsKZeS0BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEDgsqnc8OJg6Ln3AAAAAElFTkSuQmCC"56    )!5758    /// Model IDs that are documented rolling aliases — they resolve on the59    /// chat endpoint but never appear in the provider's /models listing.60    static let knownUnlistedAliases: Set<String> = [61        "grok-4.20", "grok-4.20-non-reasoning", "grok-code-fast-1",62    ]6364    // MARK: - Entry6566    static func run(arguments: [String]) async -> Int32 {67        let only = value(after: "--provider", in: arguments).flatMap { ProviderID(rawValue: $0) }68        let quick = arguments.contains("--quick")69        let providers = ProviderID.builtIn.filter { only == nil || $0 == only }7071        let catalog = await MainActor.run { ModelCatalog() }72        let models = await MainActor.run { catalog.builtIn }7374        print("Zyquo Cloud verification — \(providers.count) providers, \(models.count) catalog models\n")7576        var missingKeys: [ProviderID] = []77        for provider in providers where apiKey(for: provider) == nil {78            missingKeys.append(provider)79        }80        if !missingKeys.isEmpty {81            print("⚠️  No key in environment for: \(missingKeys.map(\.rawValue).joined(separator: ", ")) — skipped\n")82        }8384        let results = await withTaskGroup(of: [TestResult].self) { group in85            for provider in providers {86                guard let key = apiKey(for: provider) else { continue }87                let providerModels = models.filter { $0.provider == provider }88                group.addTask {89                    await verify(provider: provider, key: key, models: providerModels, quick: quick)90                }91            }92            var all: [TestResult] = []93            for await batch in group { all.append(contentsOf: batch) }94            return all95        }9697        report(results: results, skipped: missingKeys)98        let failures = results.filter { !$0.passed }.count99        return failures == 0 ? 0 : 1100    }101102    // MARK: - Per-provider verification103104    private static func verify(105        provider: ProviderID, key: String, models: [AIModel], quick: Bool106    ) async -> [TestResult] {107        var results: [TestResult] = []108        let client = ProviderRegistry.client(for: provider)109110        // 1. /models diff111        if provider.supportsModelListing {112            let start = Date()113            do {114                let live = Set(try await client.listModelIDs(apiKey: key))115                let missing = models.map(\.id).filter {116                    !live.contains($0) && !knownUnlistedAliases.contains($0)117                }118                results.append(TestResult(119                    provider: provider, model: "—", test: "models",120                    passed: missing.isEmpty,121                    latency: Date().timeIntervalSince(start),122                    detail: missing.isEmpty123                        ? "\(live.count) live"124                        : "catalog IDs not live: \(missing.joined(separator: ", "))"125                ))126            } catch {127                results.append(TestResult(128                    provider: provider, model: "—", test: "models",129                    passed: false, latency: nil, detail: error.localizedDescription130                ))131            }132        }133134        // 2. "OK" sweep over every chat model (first model only in --quick mode)135        let sweep = quick ? Array(models.prefix(1)) : models136        // Serial for strict-rate-limit providers, else limited concurrency.137        let serial = provider == .cerebras || provider == .mistral138        if serial {139            for model in sweep {140                results.append(await okTest(client: client, model: model, key: key))141                if provider == .cerebras {142                    try? await Task.sleep(nanoseconds: 13_000_000_000) // free tier: 5 req/min143                } else {144                    try? await Task.sleep(nanoseconds: 1_100_000_000)145                }146            }147        } else {148            results.append(contentsOf: await limitedConcurrent(sweep, limit: 3) { model in149                await okTest(client: client, model: model, key: key)150            })151        }152153        // 3. Streaming test on the first passing model154        if let streamModel = sweep.first(where: { model in155            results.contains { $0.model == model.id && $0.test == "chat" && $0.passed }156        }) ?? sweep.first {157            results.append(await streamTest(client: client, model: streamModel, key: key))158        }159160        // 4. Vision test on the first vision-capable model161        if let visionModel = sweep.first(where: { $0.capabilities.vision && !$0.isLegacy }) {162            results.append(await visionTest(client: client, model: visionModel, key: key))163        }164165        return results166    }167168    // MARK: - Individual tests169170    private static func okTest(client: ProviderClient, model: AIModel, key: String) async -> TestResult {171        if let reason = skipList[model.id] {172            return TestResult(173                provider: model.provider, model: model.id, test: "chat",174                passed: true, latency: nil, detail: "SKIPPED: \(reason)"175            )176        }177        let start = Date()178        var parameters = ChatParameters(maxTokens: model.capabilities.reasoning ? 4096 : 256)179        if model.parameterSupport.reasoningEffort {180            parameters.reasoningEffort = "low"181        }182        let request = ChatRequest(183            model: model,184            systemPrompt: nil,185            messages: [Message(role: .user, text: "Reply with exactly: OK")],186            parameters: parameters,187            stream: false188        )189        do {190            let reply = try await client.complete(request, apiKey: key)191            let text = reply.text.trimmingCharacters(in: .whitespacesAndNewlines)192            let passed = !text.isEmpty193            return TestResult(194                provider: model.provider, model: model.id, test: "chat",195                passed: passed,196                latency: Date().timeIntervalSince(start),197                detail: passed ? nil : "empty response"198            )199        } catch {200            return TestResult(201                provider: model.provider, model: model.id, test: "chat",202                passed: false, latency: Date().timeIntervalSince(start),203                detail: error.localizedDescription204            )205        }206    }207208    private static func streamTest(client: ProviderClient, model: AIModel, key: String) async -> TestResult {209        let start = Date()210        var parameters = ChatParameters(maxTokens: model.capabilities.reasoning ? 4096 : 128)211        if model.parameterSupport.reasoningEffort { parameters.reasoningEffort = "low" }212        let request = ChatRequest(213            model: model,214            systemPrompt: nil,215            messages: [Message(role: .user, text: "Count from 1 to 5, digits separated by spaces.")],216            parameters: parameters217        )218        do {219            var deltas = 0220            var text = ""221            var sawUsage = false222            for try await event in client.streamChat(request, apiKey: key) {223                switch event {224                case .textDelta(let d): deltas += 1; text += d225                case .usage: sawUsage = true226                default: break227                }228            }229            let passed = deltas >= 1 && !text.isEmpty230            return TestResult(231                provider: model.provider, model: model.id, test: "stream",232                passed: passed,233                latency: Date().timeIntervalSince(start),234                detail: "deltas=\(deltas) usage=\(sawUsage)"235            )236        } catch {237            return TestResult(238                provider: model.provider, model: model.id, test: "stream",239                passed: false, latency: Date().timeIntervalSince(start),240                detail: error.localizedDescription241            )242        }243    }244245    private static func visionTest(client: ProviderClient, model: AIModel, key: String) async -> TestResult {246        let start = Date()247        var message = Message(role: .user, text: "What color is this image? Reply with one word.")248        message.attachments = [249            Attachment(kind: .image, fileName: "pixel.png", data: redPixelPNG, mimeType: "image/png")250        ]251        var parameters = ChatParameters(maxTokens: model.capabilities.reasoning ? 2048 : 32)252        if model.parameterSupport.reasoningEffort { parameters.reasoningEffort = "low" }253        let request = ChatRequest(254            model: model, systemPrompt: nil, messages: [message],255            parameters: parameters, stream: false256        )257        do {258            let reply = try await client.complete(request, apiKey: key)259            let passed = !reply.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty260            return TestResult(261                provider: model.provider, model: model.id, test: "vision",262                passed: passed,263                latency: Date().timeIntervalSince(start),264                detail: passed ? String(reply.text.prefix(40)) : "empty response"265            )266        } catch {267            return TestResult(268                provider: model.provider, model: model.id, test: "vision",269                passed: false, latency: Date().timeIntervalSince(start),270                detail: error.localizedDescription271            )272        }273    }274275    // MARK: - Reporting276277    private static func report(results: [TestResult], skipped: [ProviderID]) {278        var lines: [String] = []279        lines.append("<!--")280        lines.append("  VERIFICATION.md")281        lines.append("  Zyquo Cloud")282        lines.append("  Author: Simon-Pierre Boucher")283        lines.append("  Mail: contact@spboucher.ai")284        lines.append("-->")285        lines.append("")286        lines.append("# API Verification Results — \(ISO8601DateFormatter().string(from: Date()))")287        lines.append("")288        let failures = results.filter { !$0.passed }289        lines.append("**\(results.count) tests · \(results.count - failures.count) passed · \(failures.count) failed**")290        if !skipped.isEmpty {291            lines.append("")292            lines.append("Providers skipped (no key in environment): \(skipped.map(\.displayName).joined(separator: ", "))")293        }294        lines.append("")295        lines.append("| Provider | Model | Test | Result | Latency | Detail |")296        lines.append("|---|---|---|---|---|---|")297        let sorted = results.sorted {298            ($0.provider.rawValue, $0.test == "models" ? 0 : 1, $0.model, $0.test)299                < ($1.provider.rawValue, $1.test == "models" ? 0 : 1, $1.model, $1.test)300        }301        for result in sorted {302            let mark = result.passed ? "✅" : "❌"303            let latency = result.latency.map { String(format: "%.1fs", $0) } ?? "—"304            let detail = (result.detail ?? "").replacingOccurrences(of: "|", with: "\\|")305            lines.append("| \(result.provider.displayName) | `\(result.model)` | \(result.test) | \(mark) | \(latency) | \(detail) |")306        }307        lines.append("")308        let document = lines.joined(separator: "\n")309310        let url = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)311            .appendingPathComponent("docs/VERIFICATION.md")312        try? document.data(using: .utf8)?.write(to: url)313314        // Console summary315        print(String(repeating: "—", count: 72))316        for result in sorted where !result.passed {317            print("❌ \(result.provider.rawValue) \(result.model) [\(result.test)] — \(result.detail ?? "")")318        }319        print(String(repeating: "—", count: 72))320        print("\(results.count) tests · \(results.count - failures.count) passed · \(failures.count) failed")321        print("Full table: docs/VERIFICATION.md")322    }323324    // MARK: - Helpers325326    static func apiKey(for provider: ProviderID) -> String? {327        guard let name = environmentKeys[provider] else { return nil }328        let value = ProcessInfo.processInfo.environment[name]329        return (value?.isEmpty ?? true) ? nil : value330    }331332    /// Writes environment keys into the app's encrypted vault (for GUI use).333    static func loadVault() {334        let store = SecureKeyStore()335        var loaded: [String] = []336        for (provider, _) in environmentKeys {337            if let key = apiKey(for: provider) {338                try? store.setKey(key, for: provider)339                loaded.append(provider.rawValue)340            }341        }342        print("Vault updated with keys for: \(loaded.sorted().joined(separator: ", "))")343    }344345    private static func value(after flag: String, in arguments: [String]) -> String? {346        guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else { return nil }347        return arguments[index + 1]348    }349350    private static func limitedConcurrent<T: Sendable>(351        _ models: [AIModel], limit: Int,352        _ operation: @escaping @Sendable (AIModel) async -> T353    ) async -> [T] {354        await withTaskGroup(of: (Int, T).self) { group in355            var results: [(Int, T)] = []356            var iterator = models.enumerated().makeIterator()357            var inFlight = 0358            func addNext(_ group: inout TaskGroup<(Int, T)>) {359                guard let (index, model) = iterator.next() else { return }360                inFlight += 1361                group.addTask { (index, await operation(model)) }362            }363            for _ in 0..<limit { addNext(&group) }364            while inFlight > 0 {365                guard let result = await group.next() else { break }366                inFlight -= 1367                results.append(result)368                addNext(&group)369            }370            return results.sorted { $0.0 < $1.0 }.map(\.1)371        }372    }373}374