phase7: verify harness (--verify/--load-vault), SSE empty-line fix (critical), 429/5xx backoff, Mistral effort mapping + think-chunk arrays, Gemini models/ prefix, streaming-only fallback; phase8 prep: entitlements + notarize script
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Showing 10 changed files with +650 and −26
modified
Makefile
+3 −1
@@ -80,9 +80,11 @@ release: universal | ||
| 80 | 80 | icon: |
| 81 | 81 | scripts/generate-icon.sh |
| 82 | 82 | |
| 83 | +# Full API verification with real keys (source .env.keys first or let make do it). | |
| 83 | 84 | verify: |
| 84 | 85 | swift build -c release |
| 85 | − .build/release/zyquo-verify | |
| 86 | + @if [ -f .env.keys ]; then set -a && . ./.env.keys && set +a && .build/release/zyquo-verify $(VERIFY_ARGS); \ | |
| 87 | + else .build/release/zyquo-verify $(VERIFY_ARGS); fi | |
| 86 | 88 | |
| 87 | 89 | clean: |
| 88 | 90 | rm -rf .build $(DIST) |
added
Resources/ZyquoCloud.entitlements
+18 −0
@@ -0,0 +1,18 @@ | ||
| 1 | +<?xml version="1.0" encoding="UTF-8"?> | |
| 2 | +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | |
| 3 | +<plist version="1.0"> | |
| 4 | +<dict> | |
| 5 | + <!-- | |
| 6 | + ZyquoCloud.entitlements — Zyquo Cloud | |
| 7 | + Author: Simon-Pierre Boucher / Mail: contact@spboucher.ai | |
| 8 | + | |
| 9 | + Hardened runtime is enabled at signing time (codesign --options runtime). | |
| 10 | + Zyquo Cloud needs no entitlement exceptions: outbound HTTPS to the | |
| 11 | + configured AI providers requires none outside the App Sandbox, and the | |
| 12 | + app spawns no child code, loads no plugins, and uses no JIT. Modeled on | |
| 13 | + the zyquo-term pipeline (same signing identity and notary profile). | |
| 14 | + --> | |
| 15 | + <key>com.apple.security.cs.allow-jit</key> | |
| 16 | + <false/> | |
| 17 | +</dict> | |
| 18 | +</plist> | |
added
Sources/ZyquoCloud/App/Main.swift
+31 −0
@@ -0,0 +1,31 @@ | ||
| 1 | +// | |
| 2 | +// Main.swift | |
| 3 | +// Zyquo Cloud | |
| 4 | +// | |
| 5 | +// Author: Simon-Pierre Boucher | |
| 6 | +// Mail: contact@spboucher.ai | |
| 7 | +// | |
| 8 | +// Entry point. `--verify` runs the Phase 7 API harness headlessly (reusing | |
| 9 | +// the production provider clients); `--load-vault` seeds the encrypted vault | |
| 10 | +// from environment keys; otherwise the SwiftUI app launches. | |
| 11 | +// | |
| 12 | + | |
| 13 | +import Foundation | |
| 14 | + | |
| 15 | +@main | |
| 16 | +enum Main { | |
| 17 | + static func main() async { | |
| 18 | + let arguments = CommandLine.arguments | |
| 19 | + if arguments.contains("--verify") { | |
| 20 | + let status = await VerifyHarness.run(arguments: arguments) | |
| 21 | + exit(status) | |
| 22 | + } | |
| 23 | + if arguments.contains("--load-vault") { | |
| 24 | + VerifyHarness.loadVault() | |
| 25 | + exit(0) | |
| 26 | + } | |
| 27 | + await MainActor.run { | |
| 28 | + ZyquoCloudApp.main() | |
| 29 | + } | |
| 30 | + } | |
| 31 | +} | |
modified
Sources/ZyquoCloud/App/ZyquoCloudApp.swift
+0 −1
@@ -8,7 +8,6 @@ | ||
| 8 | 8 | |
| 9 | 9 | import SwiftUI |
| 10 | 10 | |
| 11 | −@main | |
| 12 | 11 | struct ZyquoCloudApp: App { |
| 13 | 12 | @NSApplicationDelegateAdaptor(AppDelegate.self) private var appDelegate |
| 14 | 13 | @StateObject private var environment = AppEnvironment() |
modified
Sources/ZyquoCloud/Models/AIModel.swift
+3 −0
@@ -85,6 +85,9 @@ struct ParameterSupport: Codable, Hashable { | ||
| 85 | 85 | var reasoningEffort: Bool = false |
| 86 | 86 | /// Anthropic `thinking` / Qwen `enable_thinking` style explicit thinking toggle. |
| 87 | 87 | var thinkingToggle: Bool = false |
| 88 | + /// Model rejects non-streaming calls (Qwen qwq/qvq, DashScope-hosted models | |
| 89 | + /// on Together…) — `complete` aggregates a stream instead. | |
| 90 | + var requiresStreaming: Bool = false | |
| 88 | 91 | |
| 89 | 92 | static let openAIDefault = ParameterSupport(frequencyPenalty: true, presencePenalty: true) |
| 90 | 93 | } |
modified
Sources/ZyquoCloud/Providers/OpenAICompatibleClient.swift
+95 −5
@@ -136,6 +136,49 @@ struct OpenAICompatibleClient: ProviderClient { | ||
| 136 | 136 | case content, reasoning |
| 137 | 137 | case reasoningContent = "reasoning_content" |
| 138 | 138 | } |
| 139 | + | |
| 140 | + init(from decoder: Decoder) throws { | |
| 141 | + let container = try decoder.container(keyedBy: CodingKeys.self) | |
| 142 | + reasoning = try? container.decodeIfPresent(String.self, forKey: .reasoning) | |
| 143 | + reasoningContent = try? container.decodeIfPresent(String.self, forKey: .reasoningContent) | |
| 144 | + // `content` is normally a string, but Mistral's reasoning models | |
| 145 | + // return an array of chunks ({type: "thinking"|"text", …}). | |
| 146 | + if let text = try? container.decodeIfPresent(String.self, forKey: .content) { | |
| 147 | + content = text | |
| 148 | + } else if let chunks = try? container.decodeIfPresent([ContentChunk].self, forKey: .content) { | |
| 149 | + var textParts: [String] = [] | |
| 150 | + var thinkingParts: [String] = [] | |
| 151 | + for chunk in chunks { | |
| 152 | + if chunk.type == "thinking" { | |
| 153 | + thinkingParts.append(chunk.flattenedText) | |
| 154 | + } else { | |
| 155 | + textParts.append(chunk.flattenedText) | |
| 156 | + } | |
| 157 | + } | |
| 158 | + content = textParts.joined() | |
| 159 | + let thinking = thinkingParts.joined() | |
| 160 | + if !thinking.isEmpty, reasoningContent == nil { | |
| 161 | + reasoningContent = thinking | |
| 162 | + } | |
| 163 | + } | |
| 164 | + } | |
| 165 | + | |
| 166 | + /// Mistral ThinkChunk/TextChunk: {"type":"text","text":…} or | |
| 167 | + /// {"type":"thinking","thinking":[{"type":"text","text":…}]}. | |
| 168 | + struct ContentChunk: Decodable { | |
| 169 | + var type: String? | |
| 170 | + var text: String? | |
| 171 | + var thinking: [ContentChunkPart]? | |
| 172 | + | |
| 173 | + var flattenedText: String { | |
| 174 | + if let text { return text } | |
| 175 | + return (thinking ?? []).compactMap(\.text).joined() | |
| 176 | + } | |
| 177 | + } | |
| 178 | + | |
| 179 | + struct ContentChunkPart: Decodable { | |
| 180 | + var text: String? | |
| 181 | + } | |
| 139 | 182 | } |
| 140 | 183 | |
| 141 | 184 | private struct WireUsage: Decodable { |
@@ -237,7 +280,14 @@ struct OpenAICompatibleClient: ProviderClient { | ||
| 237 | 280 | } |
| 238 | 281 | if support.frequencyPenalty { wire.frequencyPenalty = params.frequencyPenalty } |
| 239 | 282 | if support.presencePenalty { wire.presencePenalty = params.presencePenalty } |
| 240 | − if support.reasoningEffort { wire.reasoningEffort = params.reasoningEffort } | |
| 283 | + if support.reasoningEffort { | |
| 284 | + // Mistral only accepts "high"/"none": map medium→high, low→none. | |
| 285 | + if providerID == .mistral, let effort = params.reasoningEffort { | |
| 286 | + wire.reasoningEffort = effort == "low" ? "none" : "high" | |
| 287 | + } else { | |
| 288 | + wire.reasoningEffort = params.reasoningEffort | |
| 289 | + } | |
| 290 | + } | |
| 241 | 291 | if support.thinkingToggle, providerID == .qwen { |
| 242 | 292 | // DashScope: enable_thinking is only legal on streaming requests. |
| 243 | 293 | if request.stream { wire.enableThinking = params.thinkingEnabled } |
@@ -318,6 +368,10 @@ struct OpenAICompatibleClient: ProviderClient { | ||
| 318 | 368 | } |
| 319 | 369 | |
| 320 | 370 | func complete(_ request: ChatRequest, apiKey: String) async throws -> Message { |
| 371 | + // Some models reject non-streaming calls — aggregate a stream instead. | |
| 372 | + if request.model.parameterSupport.requiresStreaming { | |
| 373 | + return try await completeViaStream(request, apiKey: apiKey) | |
| 374 | + } | |
| 321 | 375 | var urlReq = try urlRequest(path: "chat/completions", apiKey: apiKey) |
| 322 | 376 | var plainRequest = request |
| 323 | 377 | plainRequest.stream = false |
@@ -350,13 +404,49 @@ struct OpenAICompatibleClient: ProviderClient { | ||
| 350 | 404 | let urlReq = try urlRequest(path: "models", apiKey: apiKey, method: "GET") |
| 351 | 405 | let data = try await StreamingService.getJSON(urlReq, provider: providerID) |
| 352 | 406 | // Together returns a bare array; everyone else wraps in {"data": […]}. |
| 407 | + // Gemini's compat endpoint prefixes IDs with "models/" — normalize. | |
| 408 | + let ids: [String] | |
| 353 | 409 | if let list = try? JSONDecoder().decode(WireModelList.self, from: data) { |
| 354 | − return list.data.map(\.id) | |
| 410 | + ids = list.data.map(\.id) | |
| 411 | + } else if let bare = try? JSONDecoder().decode([WireModelEntry].self, from: data) { | |
| 412 | + ids = bare.map(\.id) | |
| 413 | + } else { | |
| 414 | + throw ProviderError.invalidResponse(providerID, detail: "unrecognized /models response shape") | |
| 355 | 415 | } |
| 356 | − if let bare = try? JSONDecoder().decode([WireModelEntry].self, from: data) { | |
| 357 | − return bare.map(\.id) | |
| 416 | + return ids.map { $0.hasPrefix("models/") ? String($0.dropFirst(7)) : $0 } | |
| 417 | + } | |
| 418 | + | |
| 419 | + /// Non-streaming result assembled from the streaming endpoint, for models | |
| 420 | + /// that only support `stream: true`. | |
| 421 | + private func completeViaStream(_ request: ChatRequest, apiKey: String) async throws -> Message { | |
| 422 | + var text = "" | |
| 423 | + var reasoning = "" | |
| 424 | + var citations: [Citation] = [] | |
| 425 | + var usage: TokenUsage? | |
| 426 | + for try await event in streamChat(request, apiKey: apiKey) { | |
| 427 | + switch event { | |
| 428 | + case .textDelta(let delta): text += delta | |
| 429 | + case .reasoningDelta(let delta): reasoning += delta | |
| 430 | + case .citations(let c): citations = c | |
| 431 | + case .usage(let u): usage = u | |
| 432 | + case .finished: break | |
| 433 | + } | |
| 358 | 434 | } |
| 359 | − throw ProviderError.invalidResponse(providerID, detail: "unrecognized /models response shape") | |
| 435 | + var message = Message( | |
| 436 | + role: .assistant, | |
| 437 | + text: text, | |
| 438 | + reasoning: reasoning.isEmpty ? nil : reasoning, | |
| 439 | + citations: citations, | |
| 440 | + modelID: request.model.id, | |
| 441 | + provider: providerID | |
| 442 | + ) | |
| 443 | + if let usage { | |
| 444 | + message.usage = usage | |
| 445 | + message.estimatedCost = request.model.pricing?.cost( | |
| 446 | + inputTokens: usage.inputTokens, outputTokens: usage.outputTokens | |
| 447 | + ) | |
| 448 | + } | |
| 449 | + return message | |
| 360 | 450 | } |
| 361 | 451 | |
| 362 | 452 | // MARK: - Helpers |
modified
Sources/ZyquoCloud/Services/StreamingService.swift
+49 −17
@@ -76,14 +76,31 @@ enum StreamingService { | ||
| 76 | 76 | for try await byte in bytes { body.append(byte) } |
| 77 | 77 | throw ProviderError.from(status: http.statusCode, body: body, provider: provider) |
| 78 | 78 | } |
| 79 | + // NOTE: AsyncBytes.lines skips empty lines, which are the | |
| 80 | + // SSE event separators — split manually to preserve them. | |
| 79 | 81 | var parser = SSEParser() |
| 80 | − for try await line in bytes.lines { | |
| 82 | + var lineBuffer = Data() | |
| 83 | + for try await byte in bytes { | |
| 81 | 84 | if Task.isCancelled { break } |
| 85 | + if byte == 0x0A { // \n | |
| 86 | + if lineBuffer.last == 0x0D { lineBuffer.removeLast() } // \r\n | |
| 87 | + let line = String(decoding: lineBuffer, as: UTF8.self) | |
| 88 | + lineBuffer.removeAll(keepingCapacity: true) | |
| 89 | + if let event = parser.consume(line: line) { | |
| 90 | + continuation.yield(event) | |
| 91 | + } | |
| 92 | + } else { | |
| 93 | + lineBuffer.append(byte) | |
| 94 | + } | |
| 95 | + } | |
| 96 | + // Flush a trailing line + event if the stream ended | |
| 97 | + // without a final newline / blank separator. | |
| 98 | + if !lineBuffer.isEmpty { | |
| 99 | + let line = String(decoding: lineBuffer, as: UTF8.self) | |
| 82 | 100 | if let event = parser.consume(line: line) { |
| 83 | 101 | continuation.yield(event) |
| 84 | 102 | } |
| 85 | 103 | } |
| 86 | − // Flush a trailing event if the stream ended without a blank line. | |
| 87 | 104 | if let event = parser.consume(line: "") { |
| 88 | 105 | continuation.yield(event) |
| 89 | 106 | } |
@@ -100,27 +117,42 @@ enum StreamingService { | ||
| 100 | 117 | } |
| 101 | 118 | } |
| 102 | 119 | |
| 103 | − /// Non-streaming JSON POST. Returns the decoded response body data. | |
| 120 | + /// Non-streaming JSON POST with exponential backoff on 429/5xx (3 attempts). | |
| 121 | + /// Returns the response body data. | |
| 104 | 122 | static func postJSON( |
| 105 | 123 | _ request: URLRequest, |
| 106 | 124 | provider: ProviderID |
| 107 | 125 | ) async throws -> Data { |
| 108 | − do { | |
| 109 | − let (data, response) = try await session.data(for: request) | |
| 110 | − guard let http = response as? HTTPURLResponse else { | |
| 111 | − throw ProviderError.invalidResponse(provider, detail: "not an HTTP response") | |
| 112 | − } | |
| 113 | − guard (200..<300).contains(http.statusCode) else { | |
| 114 | − throw ProviderError.from(status: http.statusCode, body: data, provider: provider) | |
| 126 | + let maxAttempts = 3 | |
| 127 | + var lastError: ProviderError = .invalidResponse(provider, detail: "no attempts made") | |
| 128 | + for attempt in 1...maxAttempts { | |
| 129 | + do { | |
| 130 | + let (data, response) = try await session.data(for: request) | |
| 131 | + guard let http = response as? HTTPURLResponse else { | |
| 132 | + throw ProviderError.invalidResponse(provider, detail: "not an HTTP response") | |
| 133 | + } | |
| 134 | + guard (200..<300).contains(http.statusCode) else { | |
| 135 | + let error = ProviderError.from(status: http.statusCode, body: data, provider: provider) | |
| 136 | + if attempt < maxAttempts, http.statusCode == 429 || http.statusCode >= 500 { | |
| 137 | + lastError = error | |
| 138 | + let retryAfter = (response as? HTTPURLResponse)? | |
| 139 | + .value(forHTTPHeaderField: "Retry-After").flatMap(Double.init) | |
| 140 | + let delay = retryAfter ?? pow(2, Double(attempt)) * 2 // 4s, 8s | |
| 141 | + try await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) | |
| 142 | + continue | |
| 143 | + } | |
| 144 | + throw error | |
| 145 | + } | |
| 146 | + return data | |
| 147 | + } catch let error as ProviderError { | |
| 148 | + throw error | |
| 149 | + } catch is CancellationError { | |
| 150 | + throw ProviderError.cancelled | |
| 151 | + } catch { | |
| 152 | + throw ProviderError.networkError(underlying: error) | |
| 115 | 153 | } |
| 116 | − return data | |
| 117 | − } catch let error as ProviderError { | |
| 118 | − throw error | |
| 119 | − } catch is CancellationError { | |
| 120 | − throw ProviderError.cancelled | |
| 121 | − } catch { | |
| 122 | − throw ProviderError.networkError(underlying: error) | |
| 123 | 154 | } |
| 155 | + throw lastError | |
| 124 | 156 | } |
| 125 | 157 | |
| 126 | 158 | /// GET returning decoded JSON data, with the same error mapping. |
added
Sources/ZyquoCloud/Verify/VerifyHarness.swift
+373 −0
@@ -0,0 +1,373 @@ | ||
| 1 | +// | |
| 2 | +// VerifyHarness.swift | |
| 3 | +// Zyquo Cloud | |
| 4 | +// | |
| 5 | +// Author: Simon-Pierre Boucher | |
| 6 | +// Mail: contact@spboucher.ai | |
| 7 | +// | |
| 8 | +// Phase 7 API verification: exercises the exact production provider clients | |
| 9 | +// against live APIs with real keys. Run via `ZyquoCloud --verify` (wrapped by | |
| 10 | +// the zyquo-verify target and `make verify`). Keys come from environment | |
| 11 | +// variables (source .env.keys); they are never logged or persisted here. | |
| 12 | +// | |
| 13 | +// Per provider: /models diff vs catalog, a minimal "Reply with exactly: OK" | |
| 14 | +// completion on EVERY catalog chat model, a streaming test, and a vision test | |
| 15 | +// where supported. Results: stdout table + docs/VERIFICATION.md. | |
| 16 | +// | |
| 17 | + | |
| 18 | +import Foundation | |
| 19 | + | |
| 20 | +enum VerifyHarness { | |
| 21 | + // MARK: - Result model | |
| 22 | + | |
| 23 | + struct TestResult { | |
| 24 | + let provider: ProviderID | |
| 25 | + let model: String | |
| 26 | + let test: String | |
| 27 | + let passed: Bool | |
| 28 | + let latency: TimeInterval? | |
| 29 | + let detail: String? | |
| 30 | + } | |
| 31 | + | |
| 32 | + static let environmentKeys: [ProviderID: String] = [ | |
| 33 | + .openai: "OPENAI_API_KEY", | |
| 34 | + .anthropic: "ANTHROPIC_API_KEY", | |
| 35 | + .xai: "XAI_API_KEY", | |
| 36 | + .mistral: "MISTRAL_API_KEY", | |
| 37 | + .gemini: "GEMINI_API_KEY", | |
| 38 | + .qwen: "DASHSCOPE_API_KEY", | |
| 39 | + .deepseek: "DEEPSEEK_API_KEY", | |
| 40 | + .kimi: "MOONSHOT_API_KEY", | |
| 41 | + .perplexity: "PERPLEXITY_API_KEY", | |
| 42 | + .together: "TOGETHER_API_KEY", | |
| 43 | + .deepinfra: "DEEPINFRA_API_KEY", | |
| 44 | + .cerebras: "CEREBRAS_API_KEY", | |
| 45 | + ] | |
| 46 | + | |
| 47 | + /// Long-running or per-request-billed models excluded from the bulk "OK" | |
| 48 | + /// sweep (each documented in docs/VERIFICATION.md). | |
| 49 | + static let skipList: [String: String] = [ | |
| 50 | + "sonar-deep-research": "multi-minute agentic research runs; verified via docs only", | |
| 51 | + ] | |
| 52 | + | |
| 53 | + /// 64×64 red PNG for vision tests (several providers reject tiny images). | |
| 54 | + static let redPixelPNG = Data(base64Encoded: | |
| 55 | + "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PQQkAAAgAsetfWiP4FgYrsKZeS0BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEDgsqnc8OJg6Ln3AAAAAElFTkSuQmCC" | |
| 56 | + )! | |
| 57 | + | |
| 58 | + /// Model IDs that are documented rolling aliases — they resolve on the | |
| 59 | + /// chat endpoint but never appear in the provider's /models listing. | |
| 60 | + static let knownUnlistedAliases: Set<String> = [ | |
| 61 | + "grok-4.20", "grok-4.20-non-reasoning", "grok-code-fast-1", | |
| 62 | + ] | |
| 63 | + | |
| 64 | + // MARK: - Entry | |
| 65 | + | |
| 66 | + static func run(arguments: [String]) async -> Int32 { | |
| 67 | + let only = value(after: "--provider", in: arguments).flatMap { ProviderID(rawValue: $0) } | |
| 68 | + let quick = arguments.contains("--quick") | |
| 69 | + let providers = ProviderID.builtIn.filter { only == nil || $0 == only } | |
| 70 | + | |
| 71 | + let catalog = await MainActor.run { ModelCatalog() } | |
| 72 | + let models = await MainActor.run { catalog.builtIn } | |
| 73 | + | |
| 74 | + print("Zyquo Cloud verification — \(providers.count) providers, \(models.count) catalog models\n") | |
| 75 | + | |
| 76 | + var missingKeys: [ProviderID] = [] | |
| 77 | + for provider in providers where apiKey(for: provider) == nil { | |
| 78 | + missingKeys.append(provider) | |
| 79 | + } | |
| 80 | + if !missingKeys.isEmpty { | |
| 81 | + print("⚠️ No key in environment for: \(missingKeys.map(\.rawValue).joined(separator: ", ")) — skipped\n") | |
| 82 | + } | |
| 83 | + | |
| 84 | + let results = await withTaskGroup(of: [TestResult].self) { group in | |
| 85 | + for provider in providers { | |
| 86 | + guard let key = apiKey(for: provider) else { continue } | |
| 87 | + let providerModels = models.filter { $0.provider == provider } | |
| 88 | + group.addTask { | |
| 89 | + await verify(provider: provider, key: key, models: providerModels, quick: quick) | |
| 90 | + } | |
| 91 | + } | |
| 92 | + var all: [TestResult] = [] | |
| 93 | + for await batch in group { all.append(contentsOf: batch) } | |
| 94 | + return all | |
| 95 | + } | |
| 96 | + | |
| 97 | + report(results: results, skipped: missingKeys) | |
| 98 | + let failures = results.filter { !$0.passed }.count | |
| 99 | + return failures == 0 ? 0 : 1 | |
| 100 | + } | |
| 101 | + | |
| 102 | + // MARK: - Per-provider verification | |
| 103 | + | |
| 104 | + private static func verify( | |
| 105 | + provider: ProviderID, key: String, models: [AIModel], quick: Bool | |
| 106 | + ) async -> [TestResult] { | |
| 107 | + var results: [TestResult] = [] | |
| 108 | + let client = ProviderRegistry.client(for: provider) | |
| 109 | + | |
| 110 | + // 1. /models diff | |
| 111 | + if provider.supportsModelListing { | |
| 112 | + let start = Date() | |
| 113 | + do { | |
| 114 | + let live = Set(try await client.listModelIDs(apiKey: key)) | |
| 115 | + let missing = models.map(\.id).filter { | |
| 116 | + !live.contains($0) && !knownUnlistedAliases.contains($0) | |
| 117 | + } | |
| 118 | + results.append(TestResult( | |
| 119 | + provider: provider, model: "—", test: "models", | |
| 120 | + passed: missing.isEmpty, | |
| 121 | + latency: Date().timeIntervalSince(start), | |
| 122 | + detail: missing.isEmpty | |
| 123 | + ? "\(live.count) live" | |
| 124 | + : "catalog IDs not live: \(missing.joined(separator: ", "))" | |
| 125 | + )) | |
| 126 | + } catch { | |
| 127 | + results.append(TestResult( | |
| 128 | + provider: provider, model: "—", test: "models", | |
| 129 | + passed: false, latency: nil, detail: error.localizedDescription | |
| 130 | + )) | |
| 131 | + } | |
| 132 | + } | |
| 133 | + | |
| 134 | + // 2. "OK" sweep over every chat model (first model only in --quick mode) | |
| 135 | + let sweep = quick ? Array(models.prefix(1)) : models | |
| 136 | + // Serial for strict-rate-limit providers, else limited concurrency. | |
| 137 | + let serial = provider == .cerebras || provider == .mistral | |
| 138 | + if serial { | |
| 139 | + for model in sweep { | |
| 140 | + results.append(await okTest(client: client, model: model, key: key)) | |
| 141 | + if provider == .cerebras { | |
| 142 | + try? await Task.sleep(nanoseconds: 13_000_000_000) // free tier: 5 req/min | |
| 143 | + } else { | |
| 144 | + try? await Task.sleep(nanoseconds: 1_100_000_000) | |
| 145 | + } | |
| 146 | + } | |
| 147 | + } else { | |
| 148 | + results.append(contentsOf: await limitedConcurrent(sweep, limit: 3) { model in | |
| 149 | + await okTest(client: client, model: model, key: key) | |
| 150 | + }) | |
| 151 | + } | |
| 152 | + | |
| 153 | + // 3. Streaming test on the first passing model | |
| 154 | + if let streamModel = sweep.first(where: { model in | |
| 155 | + results.contains { $0.model == model.id && $0.test == "chat" && $0.passed } | |
| 156 | + }) ?? sweep.first { | |
| 157 | + results.append(await streamTest(client: client, model: streamModel, key: key)) | |
| 158 | + } | |
| 159 | + | |
| 160 | + // 4. Vision test on the first vision-capable model | |
| 161 | + if let visionModel = sweep.first(where: { $0.capabilities.vision && !$0.isLegacy }) { | |
| 162 | + results.append(await visionTest(client: client, model: visionModel, key: key)) | |
| 163 | + } | |
| 164 | + | |
| 165 | + return results | |
| 166 | + } | |
| 167 | + | |
| 168 | + // MARK: - Individual tests | |
| 169 | + | |
| 170 | + private static func okTest(client: ProviderClient, model: AIModel, key: String) async -> TestResult { | |
| 171 | + if let reason = skipList[model.id] { | |
| 172 | + return TestResult( | |
| 173 | + provider: model.provider, model: model.id, test: "chat", | |
| 174 | + passed: true, latency: nil, detail: "SKIPPED: \(reason)" | |
| 175 | + ) | |
| 176 | + } | |
| 177 | + let start = Date() | |
| 178 | + var parameters = ChatParameters(maxTokens: model.capabilities.reasoning ? 4096 : 64) | |
| 179 | + if model.parameterSupport.reasoningEffort { | |
| 180 | + parameters.reasoningEffort = "low" | |
| 181 | + } | |
| 182 | + let request = ChatRequest( | |
| 183 | + model: model, | |
| 184 | + systemPrompt: nil, | |
| 185 | + messages: [Message(role: .user, text: "Reply with exactly: OK")], | |
| 186 | + parameters: parameters, | |
| 187 | + stream: false | |
| 188 | + ) | |
| 189 | + do { | |
| 190 | + let reply = try await client.complete(request, apiKey: key) | |
| 191 | + let text = reply.text.trimmingCharacters(in: .whitespacesAndNewlines) | |
| 192 | + let passed = !text.isEmpty | |
| 193 | + return TestResult( | |
| 194 | + provider: model.provider, model: model.id, test: "chat", | |
| 195 | + passed: passed, | |
| 196 | + latency: Date().timeIntervalSince(start), | |
| 197 | + detail: passed ? nil : "empty response" | |
| 198 | + ) | |
| 199 | + } catch { | |
| 200 | + return TestResult( | |
| 201 | + provider: model.provider, model: model.id, test: "chat", | |
| 202 | + passed: false, latency: Date().timeIntervalSince(start), | |
| 203 | + detail: error.localizedDescription | |
| 204 | + ) | |
| 205 | + } | |
| 206 | + } | |
| 207 | + | |
| 208 | + private static func streamTest(client: ProviderClient, model: AIModel, key: String) async -> TestResult { | |
| 209 | + let start = Date() | |
| 210 | + var parameters = ChatParameters(maxTokens: model.capabilities.reasoning ? 4096 : 128) | |
| 211 | + if model.parameterSupport.reasoningEffort { parameters.reasoningEffort = "low" } | |
| 212 | + let request = ChatRequest( | |
| 213 | + model: model, | |
| 214 | + systemPrompt: nil, | |
| 215 | + messages: [Message(role: .user, text: "Count from 1 to 5, digits separated by spaces.")], | |
| 216 | + parameters: parameters | |
| 217 | + ) | |
| 218 | + do { | |
| 219 | + var deltas = 0 | |
| 220 | + var text = "" | |
| 221 | + var sawUsage = false | |
| 222 | + for try await event in client.streamChat(request, apiKey: key) { | |
| 223 | + switch event { | |
| 224 | + case .textDelta(let d): deltas += 1; text += d | |
| 225 | + case .usage: sawUsage = true | |
| 226 | + default: break | |
| 227 | + } | |
| 228 | + } | |
| 229 | + let passed = deltas >= 1 && !text.isEmpty | |
| 230 | + return TestResult( | |
| 231 | + provider: model.provider, model: model.id, test: "stream", | |
| 232 | + passed: passed, | |
| 233 | + latency: Date().timeIntervalSince(start), | |
| 234 | + detail: "deltas=\(deltas) usage=\(sawUsage)" | |
| 235 | + ) | |
| 236 | + } catch { | |
| 237 | + return TestResult( | |
| 238 | + provider: model.provider, model: model.id, test: "stream", | |
| 239 | + passed: false, latency: Date().timeIntervalSince(start), | |
| 240 | + detail: error.localizedDescription | |
| 241 | + ) | |
| 242 | + } | |
| 243 | + } | |
| 244 | + | |
| 245 | + private static func visionTest(client: ProviderClient, model: AIModel, key: String) async -> TestResult { | |
| 246 | + let start = Date() | |
| 247 | + var message = Message(role: .user, text: "What color is this image? Reply with one word.") | |
| 248 | + message.attachments = [ | |
| 249 | + Attachment(kind: .image, fileName: "pixel.png", data: redPixelPNG, mimeType: "image/png") | |
| 250 | + ] | |
| 251 | + var parameters = ChatParameters(maxTokens: model.capabilities.reasoning ? 2048 : 32) | |
| 252 | + if model.parameterSupport.reasoningEffort { parameters.reasoningEffort = "low" } | |
| 253 | + let request = ChatRequest( | |
| 254 | + model: model, systemPrompt: nil, messages: [message], | |
| 255 | + parameters: parameters, stream: false | |
| 256 | + ) | |
| 257 | + do { | |
| 258 | + let reply = try await client.complete(request, apiKey: key) | |
| 259 | + let passed = !reply.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty | |
| 260 | + return TestResult( | |
| 261 | + provider: model.provider, model: model.id, test: "vision", | |
| 262 | + passed: passed, | |
| 263 | + latency: Date().timeIntervalSince(start), | |
| 264 | + detail: passed ? String(reply.text.prefix(40)) : "empty response" | |
| 265 | + ) | |
| 266 | + } catch { | |
| 267 | + return TestResult( | |
| 268 | + provider: model.provider, model: model.id, test: "vision", | |
| 269 | + passed: false, latency: Date().timeIntervalSince(start), | |
| 270 | + detail: error.localizedDescription | |
| 271 | + ) | |
| 272 | + } | |
| 273 | + } | |
| 274 | + | |
| 275 | + // MARK: - Reporting | |
| 276 | + | |
| 277 | + private static func report(results: [TestResult], skipped: [ProviderID]) { | |
| 278 | + var lines: [String] = [] | |
| 279 | + lines.append("<!--") | |
| 280 | + lines.append(" VERIFICATION.md") | |
| 281 | + lines.append(" Zyquo Cloud") | |
| 282 | + lines.append(" Author: Simon-Pierre Boucher") | |
| 283 | + lines.append(" Mail: contact@spboucher.ai") | |
| 284 | + lines.append("-->") | |
| 285 | + lines.append("") | |
| 286 | + lines.append("# API Verification Results — \(ISO8601DateFormatter().string(from: Date()))") | |
| 287 | + lines.append("") | |
| 288 | + let failures = results.filter { !$0.passed } | |
| 289 | + lines.append("**\(results.count) tests · \(results.count - failures.count) passed · \(failures.count) failed**") | |
| 290 | + if !skipped.isEmpty { | |
| 291 | + lines.append("") | |
| 292 | + lines.append("Providers skipped (no key in environment): \(skipped.map(\.displayName).joined(separator: ", "))") | |
| 293 | + } | |
| 294 | + lines.append("") | |
| 295 | + lines.append("| Provider | Model | Test | Result | Latency | Detail |") | |
| 296 | + lines.append("|---|---|---|---|---|---|") | |
| 297 | + let sorted = results.sorted { | |
| 298 | + ($0.provider.rawValue, $0.test == "models" ? 0 : 1, $0.model, $0.test) | |
| 299 | + < ($1.provider.rawValue, $1.test == "models" ? 0 : 1, $1.model, $1.test) | |
| 300 | + } | |
| 301 | + for result in sorted { | |
| 302 | + let mark = result.passed ? "✅" : "❌" | |
| 303 | + let latency = result.latency.map { String(format: "%.1fs", $0) } ?? "—" | |
| 304 | + let detail = (result.detail ?? "").replacingOccurrences(of: "|", with: "\\|") | |
| 305 | + lines.append("| \(result.provider.displayName) | `\(result.model)` | \(result.test) | \(mark) | \(latency) | \(detail) |") | |
| 306 | + } | |
| 307 | + lines.append("") | |
| 308 | + let document = lines.joined(separator: "\n") | |
| 309 | + | |
| 310 | + let url = URL(fileURLWithPath: FileManager.default.currentDirectoryPath) | |
| 311 | + .appendingPathComponent("docs/VERIFICATION.md") | |
| 312 | + try? document.data(using: .utf8)?.write(to: url) | |
| 313 | + | |
| 314 | + // Console summary | |
| 315 | + print(String(repeating: "—", count: 72)) | |
| 316 | + for result in sorted where !result.passed { | |
| 317 | + print("❌ \(result.provider.rawValue) \(result.model) [\(result.test)] — \(result.detail ?? "")") | |
| 318 | + } | |
| 319 | + print(String(repeating: "—", count: 72)) | |
| 320 | + print("\(results.count) tests · \(results.count - failures.count) passed · \(failures.count) failed") | |
| 321 | + print("Full table: docs/VERIFICATION.md") | |
| 322 | + } | |
| 323 | + | |
| 324 | + // MARK: - Helpers | |
| 325 | + | |
| 326 | + static func apiKey(for provider: ProviderID) -> String? { | |
| 327 | + guard let name = environmentKeys[provider] else { return nil } | |
| 328 | + let value = ProcessInfo.processInfo.environment[name] | |
| 329 | + return (value?.isEmpty ?? true) ? nil : value | |
| 330 | + } | |
| 331 | + | |
| 332 | + /// Writes environment keys into the app's encrypted vault (for GUI use). | |
| 333 | + static func loadVault() { | |
| 334 | + let store = SecureKeyStore() | |
| 335 | + var loaded: [String] = [] | |
| 336 | + for (provider, _) in environmentKeys { | |
| 337 | + if let key = apiKey(for: provider) { | |
| 338 | + try? store.setKey(key, for: provider) | |
| 339 | + loaded.append(provider.rawValue) | |
| 340 | + } | |
| 341 | + } | |
| 342 | + print("Vault updated with keys for: \(loaded.sorted().joined(separator: ", "))") | |
| 343 | + } | |
| 344 | + | |
| 345 | + private static func value(after flag: String, in arguments: [String]) -> String? { | |
| 346 | + guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else { return nil } | |
| 347 | + return arguments[index + 1] | |
| 348 | + } | |
| 349 | + | |
| 350 | + private static func limitedConcurrent<T: Sendable>( | |
| 351 | + _ models: [AIModel], limit: Int, | |
| 352 | + _ operation: @escaping @Sendable (AIModel) async -> T | |
| 353 | + ) async -> [T] { | |
| 354 | + await withTaskGroup(of: (Int, T).self) { group in | |
| 355 | + var results: [(Int, T)] = [] | |
| 356 | + var iterator = models.enumerated().makeIterator() | |
| 357 | + var inFlight = 0 | |
| 358 | + func addNext(_ group: inout TaskGroup<(Int, T)>) { | |
| 359 | + guard let (index, model) = iterator.next() else { return } | |
| 360 | + inFlight += 1 | |
| 361 | + group.addTask { (index, await operation(model)) } | |
| 362 | + } | |
| 363 | + for _ in 0..<limit { addNext(&group) } | |
| 364 | + while inFlight > 0 { | |
| 365 | + guard let result = await group.next() else { break } | |
| 366 | + inFlight -= 1 | |
| 367 | + results.append(result) | |
| 368 | + addNext(&group) | |
| 369 | + } | |
| 370 | + return results.sorted { $0.0 < $1.0 }.map(\.1) | |
| 371 | + } | |
| 372 | + } | |
| 373 | +} | |
modified
Sources/ZyquoVerify/main.swift
+22 −2
@@ -5,7 +5,27 @@ | ||
| 5 | 5 | // Author: Simon-Pierre Boucher |
| 6 | 6 | // Mail: contact@spboucher.ai |
| 7 | 7 | // |
| 8 | −// zyquo-verify — Phase 7 API verification harness (implemented in Phase 7). | |
| 8 | +// zyquo-verify — thin launcher for the Phase 7 harness. The harness itself | |
| 9 | +// lives inside the ZyquoCloud binary (`--verify`) so it exercises the exact | |
| 10 | +// production provider clients; this target execs the sibling binary. | |
| 9 | 11 | // |
| 10 | 12 | |
| 11 | −print("zyquo-verify: harness not yet implemented (Phase 7)") | |
| 13 | +import Foundation | |
| 14 | + | |
| 15 | +let selfURL = URL(fileURLWithPath: CommandLine.arguments[0]).resolvingSymlinksInPath() | |
| 16 | +let appBinary = selfURL.deletingLastPathComponent().appendingPathComponent("ZyquoCloud") | |
| 17 | + | |
| 18 | +guard FileManager.default.isExecutableFile(atPath: appBinary.path) else { | |
| 19 | + FileHandle.standardError.write(Data( | |
| 20 | + "error: ZyquoCloud binary not found next to zyquo-verify (build with `swift build`)\n".utf8 | |
| 21 | + )) | |
| 22 | + exit(1) | |
| 23 | +} | |
| 24 | + | |
| 25 | +let process = Process() | |
| 26 | +process.executableURL = appBinary | |
| 27 | +process.arguments = ["--verify"] + CommandLine.arguments.dropFirst() | |
| 28 | +process.environment = ProcessInfo.processInfo.environment | |
| 29 | +try process.run() | |
| 30 | +process.waitUntilExit() | |
| 31 | +exit(process.terminationStatus) | |
added
scripts/notarize.sh
+56 −0
@@ -0,0 +1,56 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +# | |
| 3 | +# notarize.sh | |
| 4 | +# Zyquo Cloud | |
| 5 | +# | |
| 6 | +# Author: Simon-Pierre Boucher | |
| 7 | +# Mail: contact@spboucher.ai | |
| 8 | +# | |
| 9 | +# Developer ID signing + notarization + stapling (Phase 8), reusing the | |
| 10 | +# proven zyquo-term pipeline: identity "Developer ID Application: | |
| 11 | +# Simon-Pierre Boucher (3YM54G49SN)" and notarytool keychain profile | |
| 12 | +# "MacLustr-Notarize". | |
| 13 | +# | |
| 14 | +# Usage: notarize.sh <app-dir> <identity> <keychain-profile> <entitlements> | |
| 15 | +# | |
| 16 | +set -euo pipefail | |
| 17 | + | |
| 18 | +APP_DIR="$1"; IDENTITY="$2"; PROFILE="$3"; ENTITLEMENTS="$4" | |
| 19 | +DIST="$(dirname "$APP_DIR")" | |
| 20 | +APP_NAME="$(basename "$APP_DIR" .app)" | |
| 21 | +ZIP_PATH="$DIST/$APP_NAME.zip" | |
| 22 | +DMG_PATH="$DIST/ZyquoCloud.dmg" | |
| 23 | + | |
| 24 | +echo "=== Signing (Developer ID, hardened runtime) ===" | |
| 25 | +codesign --force --options runtime --timestamp \ | |
| 26 | + --entitlements "$ENTITLEMENTS" \ | |
| 27 | + --sign "$IDENTITY" "$APP_DIR" | |
| 28 | +codesign --verify --deep --strict --verbose=2 "$APP_DIR" | |
| 29 | +echo "Signature valid." | |
| 30 | + | |
| 31 | +echo "=== Notarizing app (profile: $PROFILE) ===" | |
| 32 | +rm -f "$ZIP_PATH" | |
| 33 | +ditto -c -k --keepParent "$APP_DIR" "$ZIP_PATH" | |
| 34 | +xcrun notarytool submit "$ZIP_PATH" --keychain-profile "$PROFILE" --wait | |
| 35 | +xcrun stapler staple "$APP_DIR" | |
| 36 | +xcrun stapler validate "$APP_DIR" | |
| 37 | + | |
| 38 | +echo "=== Gatekeeper check ===" | |
| 39 | +spctl -a -vv "$APP_DIR" | |
| 40 | + | |
| 41 | +echo "=== Building distributable DMG ===" | |
| 42 | +rm -f "$DMG_PATH" | |
| 43 | +DMG_TEMP="$DIST/dmg_temp" | |
| 44 | +rm -rf "$DMG_TEMP" | |
| 45 | +mkdir -p "$DMG_TEMP" | |
| 46 | +cp -R "$APP_DIR" "$DMG_TEMP/" | |
| 47 | +ln -s /Applications "$DMG_TEMP/Applications" | |
| 48 | +hdiutil create -volname "$APP_NAME" -srcfolder "$DMG_TEMP" -ov -format UDZO "$DMG_PATH" | |
| 49 | +rm -rf "$DMG_TEMP" | |
| 50 | +codesign --force --sign "$IDENTITY" --timestamp "$DMG_PATH" | |
| 51 | + | |
| 52 | +echo "=== Notarizing DMG ===" | |
| 53 | +xcrun notarytool submit "$DMG_PATH" --keychain-profile "$PROFILE" --wait | |
| 54 | +xcrun stapler staple "$DMG_PATH" | |
| 55 | + | |
| 56 | +echo "Release complete: $APP_DIR (stapled) + $DMG_PATH (notarized, stapled)" | |
| 57 | ||