spb/zyquo-agent Public MIT
The autonomous agent that actually operates your Mac — plans, runs real commands, verifies its own work.
Swift 94.7%
Shell 4.1%
Python 0.7%
Makefile 0.5%
1//2// VerifyHarness.swift3// Zyquo Agent4//5// Author: Simon-Pierre Boucher6// Mail: contact@spboucher.ai7//8// Phase 7.1 — provider tool-calling verification, run against LIVE APIs with9// the exact production provider clients. For every agent-capable model whose10// provider has an API key available (environment or vault), the harness:11//12// 1. Streams a chat request offering ONE tool (`list_files`) and asserts the13// model emits a tool call with the right name, syntactically valid JSON14// arguments, and a normalized StopReason of `.toolUse`.15// 2. Threads a canned tool_result back in the provider's wire format and16// asserts the model produces a final text answer that mentions at least17// one of the listed file names, with StopReason `.endTurn`.18// 3. Asserts streaming actually streamed (more than one text/argument delta19// across the exchange, not a single blob).20//21// Results are printed live and written as a markdown table to22// docs/VERIFICATION.md. Providers run sequentially, models within a provider23// sequentially, with a polite delay between calls. Keys are never logged.24//25// Usage: ZyquoAgent --verify [--provider <id>] [--model <id | provider/id>]26//2728import Foundation2930enum VerifyHarness {3132 // MARK: - Constants3334 /// The single tool offered to every model. {path?: string} — an empty35 /// arguments object is a perfectly valid call.36 static let listFilesTool = ToolSpec(37 name: "list_files",38 description: "List the files in the current workspace directory.",39 parametersJSONSchema: """40 {"type":"object","properties":{"path":{"type":"string","description":"Optional subdirectory to list, relative to the workspace root."}}}41 """42 )4344 static let systemPrompt = """45 You are Zyquo Agent, an autonomous agent operating inside a workspace directory on the user's Mac. \46 You must use the provided tools to inspect the workspace — never guess or fabricate file listings. \47 Call a tool when you need information. Once you have the tool result, answer the user in plain text \48 (naming the files you found) and stop; do not call the same tool again.49 """5051 static let userPrompt = "List the files in the workspace using the tool."5253 /// Canned tool_result threaded back on step 2.54 static let cannedListing = "MEMORY.md\nnotes.txt\nresult.csv"55 static let expectedNames = ["MEMORY.md", "notes.txt", "result.csv"]5657 /// Whole-model budget (both steps + retries within a step).58 static let perModelTimeout: TimeInterval = 905960 // MARK: - Result model6162 struct ModelReport {63 let model: AIModel64 var toolCall = false65 var threading = false66 var streaming = false67 var latency: TimeInterval?68 var notes: [String] = []6970 var green: Bool { toolCall && threading && streaming }71 }7273 private struct VerifyTimeout: Error {}7475 /// Everything observed while one streamed response was consumed.76 private struct StreamOutcome {77 var text = ""78 var textDeltas = 079 var reasoningDeltas = 080 var toolCallStarts = 081 var argDeltas = 082 var toolCalls: [ToolCall] = []83 var rawReason: String?84 var stop: StopReason = .other(nil)85 }8687 // MARK: - Entry8889 static func run(arguments: [String]) async -> Int32 {90 let onlyProvider = value(after: "--provider", in: arguments).flatMap { ProviderID(rawValue: $0) }91 let onlyModel = value(after: "--model", in: arguments)9293 var models = ModelCatalogData.all.filter(\.agentCapable)94 if let onlyProvider {95 models = models.filter { $0.provider == onlyProvider }96 }97 if let onlyModel {98 models = models.filter { $0.id == onlyModel || "\($0.provider.rawValue)/\($0.id)" == onlyModel }99 if models.isEmpty {100 FileHandle.standardError.write(Data("No agent-capable model matches “\(onlyModel)”.\n".utf8))101 return 64102 }103 }104105 // Group by provider, in the canonical provider order.106 let providers = ProviderID.builtIn.filter { provider in107 models.contains { $0.provider == provider }108 }109 var keyless: [ProviderID] = []110 var reports: [ModelReport] = []111112 let total = providers.reduce(0) { count, provider in113 AgentCLI.resolveAPIKey(for: provider) == nil114 ? count115 : count + models.filter { $0.provider == provider }.count116 }117 print("Zyquo Agent — provider tool-calling verification")118 print(" \(total) agent-capable model(s) across \(providers.count) provider(s)\n")119120 for provider in providers {121 let providerModels = models.filter { $0.provider == provider }122 guard let key = AgentCLI.resolveAPIKey(for: provider) else {123 keyless.append(provider)124 print("⚠️ \(provider.displayName): no API key in environment or vault — \(providerModels.count) model(s) skipped")125 continue126 }127 let client = ProviderRegistry.client(for: provider)128 print("── \(provider.displayName) (\(providerModels.count) model(s)) " + String(repeating: "─", count: 30))129 for model in providerModels {130 var report = await checkWithRetry(model: model, client: client, key: key)131 if !report.green {132 report.notes.insert("failed twice (attempt 1 + retry)", at: 0)133 }134 reports.append(report)135 printLine(report)136 try? await Task.sleep(nanoseconds: interCallDelay(for: provider))137 }138 }139140 writeReport(reports: reports, keyless: keyless)141142 let green = reports.filter(\.green).count143 print("\n\(reports.count) model(s) tested · \(green) green · \(reports.count - green) failed")144 print("Full table: docs/VERIFICATION.md")145 return green == reports.count ? 0 : 1146 }147148 // MARK: - Per-model check149150 /// One attempt; on any failure, a single automatic retry (transient151 /// hiccups, rate-limit blips). Notes reflect the final attempt.152 private static func checkWithRetry(model: AIModel, client: any ProviderClient, key: String) async -> ModelReport {153 var start = Date()154 var first = await check(model: model, client: client, key: key)155 first.latency = Date().timeIntervalSince(start)156 if first.green { return first }157 try? await Task.sleep(nanoseconds: 2_000_000_000)158 start = Date()159 var second = await check(model: model, client: client, key: key)160 second.latency = Date().timeIntervalSince(start)161 if second.green {162 second.notes.append("passed on retry")163 }164 return second165 }166167 private static func check(model: AIModel, client: any ProviderClient, key: String) async -> ModelReport {168 var report = ModelReport(model: model)169 let start = Date()170171 var parameters = ChatParameters(maxTokens: 300)172 if model.parameterSupport.reasoningEffort { parameters.reasoningEffort = "low" }173 if model.parameterSupport.thinkingToggle { parameters.thinkingEnabled = false }174175 // ---- Step 1: the model must call list_files -------------------------176 let step1 = ChatRequest(177 model: model,178 systemPrompt: systemPrompt,179 messages: [Message(role: .user, text: userPrompt)],180 parameters: parameters,181 tools: [listFilesTool],182 toolChoice: .auto183 )184 let outcome1: StreamOutcome185 do {186 outcome1 = try await collectPatiently(client, step1, apiKey: key, deadline: perModelTimeout)187 } catch {188 report.notes.append("step 1: \(describe(error))")189 return report190 }191192 let calls = outcome1.toolCalls193 if calls.isEmpty {194 report.notes.append("step 1: no tool call emitted (stop=\(outcome1.rawReason ?? "nil"))"195 + (outcome1.text.isEmpty ? "" : "; text: “\(truncate(outcome1.text, 80))”"))196 if outcome1.stop == .maxTokens {197 report.notes.append("token-starved at 300 max_tokens")198 }199 return report200 }201 var callProblems: [String] = []202 if !calls.contains(where: { $0.name == listFilesTool.name }) {203 callProblems.append("wrong tool name “\(calls.map(\.name).joined(separator: ","))”")204 }205 for call in calls where call.argumentsDictionary == nil {206 callProblems.append("arguments not valid JSON object: \(truncate(call.argumentsJSON, 80))")207 }208 if outcome1.stop != .toolUse {209 callProblems.append("stop reason not toolUse (raw=\(outcome1.rawReason ?? "nil"))")210 }211 report.toolCall = callProblems.isEmpty212 report.notes.append(contentsOf: callProblems.map { "step 1: \($0)" })213 if calls.count > 1 {214 report.notes.append("emitted \(calls.count) parallel calls")215 }216 guard report.toolCall else { return report }217218 // ---- Step 2: thread the canned tool_result back ----------------------219 let assistantTurn = Message.assistantToolCalls(calls, text: outcome1.text)220 let results = calls.map { ToolResult(toolCallID: $0.id, content: cannedListing) }221 let step2 = ChatRequest(222 model: model,223 systemPrompt: systemPrompt,224 messages: [225 Message(role: .user, text: userPrompt),226 assistantTurn,227 Message.toolResultsMessage(results),228 ],229 parameters: parameters,230 tools: [listFilesTool],231 toolChoice: .auto232 )233 let remaining = max(15, perModelTimeout - Date().timeIntervalSince(start))234 let outcome2: StreamOutcome235 do {236 outcome2 = try await collectPatiently(client, step2, apiKey: key, deadline: remaining)237 } catch {238 report.notes.append("step 2: \(describe(error))")239 return report240 }241242 var threadProblems: [String] = []243 let answer = outcome2.text.trimmingCharacters(in: .whitespacesAndNewlines)244 if !outcome2.toolCalls.isEmpty {245 threadProblems.append("called the tool again after receiving the result")246 }247 if answer.isEmpty {248 threadProblems.append("empty final answer (stop=\(outcome2.rawReason ?? "nil"))")249 } else if !expectedNames.contains(where: { answer.localizedCaseInsensitiveContains($0) }) {250 threadProblems.append("final answer names none of the files: “\(truncate(answer, 80))”")251 }252 if outcome2.stop != .endTurn {253 threadProblems.append("stop reason not endTurn (raw=\(outcome2.rawReason ?? "nil"))")254 }255 report.threading = threadProblems.isEmpty256 report.notes.append(contentsOf: threadProblems.map { "step 2: \($0)" })257258 // ---- Streaming: deltas must have arrived incrementally ---------------259 let deltas = outcome1.textDeltas + outcome1.argDeltas + outcome2.textDeltas260 report.streaming = deltas > 1261 if !report.streaming {262 report.notes.append("streaming: only \(deltas) delta event(s) — arrived as a single blob")263 }264 return report265 }266267 // MARK: - Stream consumption268269 /// `collect` with rate-limit patience: strict per-model RPM tiers and270 /// intermittent capacity waves (Moonshot's kimi-k3 429s with271 /// `engine_overloaded_error` on ~half of requests) clear after a short272 /// wait — retry up to twice with growing pauses. Other errors propagate.273 private static func collectPatiently(274 _ client: any ProviderClient,275 _ request: ChatRequest,276 apiKey: String,277 deadline: TimeInterval278 ) async throws -> StreamOutcome {279 var wait: TimeInterval = 20280 for _ in 0..<2 {281 do {282 return try await collect(client, request, apiKey: apiKey, deadline: deadline)283 } catch ProviderError.rateLimited(_, let retryAfter) {284 try await Task.sleep(nanoseconds: UInt64(max(retryAfter ?? wait, wait) * 1_000_000_000))285 wait *= 2286 }287 }288 return try await collect(client, request, apiKey: apiKey, deadline: deadline)289 }290291 /// Consumes one streamed response fully, racing a deadline. Any thrown292 /// provider/network error propagates; a deadline hit throws VerifyTimeout.293 private static func collect(294 _ client: any ProviderClient,295 _ request: ChatRequest,296 apiKey: String,297 deadline: TimeInterval298 ) async throws -> StreamOutcome {299 try await withThrowingTaskGroup(of: StreamOutcome?.self) { group in300 group.addTask {301 var out = StreamOutcome()302 for try await event in client.streamChat(request, apiKey: apiKey) {303 switch event {304 case .textDelta(let delta):305 out.text += delta306 out.textDeltas += 1307 case .reasoningDelta:308 out.reasoningDeltas += 1309 case .toolCallStarted:310 out.toolCallStarts += 1311 case .toolCallArgumentsDelta:312 out.argDeltas += 1313 case .toolCalls(let calls):314 out.toolCalls = calls315 case .finished(let reason, let stop):316 out.rawReason = reason317 out.stop = stop318 case .citations, .usage:319 break320 }321 }322 return out323 }324 group.addTask {325 try await Task.sleep(nanoseconds: UInt64(deadline * 1_000_000_000))326 return nil327 }328 guard let first = try await group.next(), let outcome = first else {329 group.cancelAll()330 throw VerifyTimeout()331 }332 group.cancelAll()333 return outcome334 }335 }336337 // MARK: - Reporting338339 private static func printLine(_ report: ModelReport) {340 let marks = [341 report.toolCall ? "tool✅" : "tool❌",342 report.threading ? "thread✅" : "thread❌",343 report.streaming ? "stream✅" : "stream❌",344 ].joined(separator: " ")345 let latency = report.latency.map { String(format: "%.1fs", $0) } ?? "—"346 var line = " \(report.green ? "✅" : "❌") \(report.model.id) \(marks) \(latency)"347 if !report.notes.isEmpty {348 line += "\n · " + report.notes.joined(separator: "\n · ")349 }350 print(line)351 }352353 private static func writeReport(reports: [ModelReport], keyless: [ProviderID]) {354 var lines: [String] = []355 lines.append("<!--")356 lines.append(" VERIFICATION.md")357 lines.append(" Zyquo Agent")358 lines.append(" Author: Simon-Pierre Boucher")359 lines.append(" Mail: contact@spboucher.ai")360 lines.append("-->")361 lines.append("")362 lines.append("# Provider Tool-Calling Verification — \(ISO8601DateFormatter().string(from: Date()))")363 lines.append("")364 lines.append("Phase 7.1 live sweep (`ZyquoAgent --verify`): every agent-capable model receives")365 lines.append("the `list_files` tool schema, must emit a valid tool call (streamed), consume the")366 lines.append("threaded `tool_result`, and produce a final answer naming the listed files.")367 lines.append("")368 let green = reports.filter(\.green).count369 lines.append("**\(reports.count) models tested · \(green) green · \(reports.count - green) failed**")370 if !keyless.isEmpty {371 lines.append("")372 lines.append("Providers skipped (no key available): \(keyless.map(\.displayName).joined(separator: ", "))")373 }374 lines.append("")375 lines.append("| Provider | Model | Tool call | Threading | Streaming | Latency | Notes |")376 lines.append("|---|---|---|---|---|---|---|")377 for report in reports {378 let latency = report.latency.map { String(format: "%.1fs", $0) } ?? "—"379 let notes = report.notes.joined(separator: "; ")380 .replacingOccurrences(of: "|", with: "\\|")381 .replacingOccurrences(of: "\n", with: " ")382 lines.append("| \(report.model.provider.displayName) | `\(report.model.id)` "383 + "| \(report.toolCall ? "✅" : "❌") | \(report.threading ? "✅" : "❌") "384 + "| \(report.streaming ? "✅" : "❌") | \(latency) | \(notes) |")385 }386 lines.append("")387 let url = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)388 .appendingPathComponent("docs/VERIFICATION.md")389 try? lines.joined(separator: "\n").data(using: .utf8)?.write(to: url)390 }391392 // MARK: - Helpers393394 /// Gentle pacing between models of one provider (Cerebras free-tier rate395 /// limits are the strictest; everyone else gets a polite second).396 private static func interCallDelay(for provider: ProviderID) -> UInt64 {397 switch provider {398 case .cerebras: return 10_000_000_000399 case .mistral: return 2_000_000_000400 default: return 1_200_000_000401 }402 }403404 private static func describe(_ error: Error) -> String {405 if error is VerifyTimeout { return "timed out after \(Int(perModelTimeout))s" }406 if let provider = error as? ProviderError {407 var text = provider.errorDescription ?? "\(provider)"408 if case .badRequest(_, let message) = provider {409 let lowered = (message ?? "").lowercased()410 if lowered.contains("model") &&411 (lowered.contains("not") || lowered.contains("exist") || lowered.contains("found")412 || lowered.contains("invalid") || lowered.contains("decommission")413 || lowered.contains("deprecat") || lowered.contains("access")) {414 text = "model id rejected by API — " + text415 }416 }417 return truncate(text, 220)418 }419 return truncate(error.localizedDescription, 220)420 }421422 private static func truncate(_ text: String, _ limit: Int) -> String {423 let flat = text.replacingOccurrences(of: "\n", with: " ")424 return flat.count > limit ? String(flat.prefix(limit)) + "…" : flat425 }426427 private static func value(after flag: String, in arguments: [String]) -> String? {428 guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else { return nil }429 return arguments[index + 1]430 }431}432