SPB Git

spb/zyquo-agent Public MIT

The autonomous agent that actually operates your Mac — plans, runs real commands, verifies its own work.

macosswiftaiagent
Swift 94.7% Shell 4.1% Python 0.7% Makefile 0.5%

phase7.1: live provider verification 77/80 green — VerifyHarness, --load-vault, Gemini thought_signature round-trip, GPT-5.4+ reasoning_effort quirk

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
simon-pierre boucher committed 11 days ago (Jul 30, 2026) parent bd18a49

Showing 5 changed files with +644 and −9

modified Sources/ZyquoAgent/App/AgentCLI.swift +27 −4
@@ -18,7 +18,12 @@
18 18 // Hidden CI smoke test: same engine, scripted MockProviderClient,
19 19 // scratch workspace, no network/keys.
20 20 //
21 −// --verify Provider tool-calling harness (arrives in Phase 7).
21 +// --verify [--provider <id>] [--model <id | provider/id>]
22 +// Phase 7.1: live provider tool-calling verification over every
23 +// agent-capable model (see Verify/VerifyHarness.swift). Writes
24 +// docs/VERIFICATION.md.
25 +//
26 +// --load-vault Imports environment API keys into the encrypted vault.
22 27 // --verify-policy PolicyEngine safety self-check (Phase 3.C).
23 28 //
24 29 // `--yes` auto-approves mode-driven approvals for scripted runs, but NEVER
@@ -40,8 +45,7 @@ enum AgentCLI {
40 45 return allPassed ? 0 : 1
41 46 }
42 47 if arguments.contains("--verify") {
43 − FileHandle.standardError.write(Data("Zyquo Agent CLI: --verify (provider tool-calling harness) arrives in Phase 7.\n".utf8))
44 − return 64
48 + return await VerifyHarness.run(arguments: arguments)
45 49 }
46 50 if arguments.contains("--run") || arguments.contains("--run-mock") {
47 51 switch parseRunOptions(arguments) {
@@ -56,8 +60,27 @@ enum AgentCLI {
56 60 return 64
57 61 }
58 62
63 + /// `--load-vault`: seeds the encrypted vault from environment variables.
64 + /// Prints stored/skipped per provider; NEVER prints any part of a key.
59 65 static func loadVault() {
60 − FileHandle.standardError.write(Data("Zyquo Agent CLI: --load-vault arrives with the Settings key-import flow.\n".utf8))
66 + let store = SecureKeyStore()
67 + let environment = ProcessInfo.processInfo.environment
68 + print("Zyquo Agent — importing API keys from the environment into the vault")
69 + for provider in ProviderID.builtIn {
70 + let names = environmentKeyNames(for: provider)
71 + guard let name = names.first(where: { !(environment[$0] ?? "").isEmpty }),
72 + let key = environment[name] else {
73 + print(" skipped \(provider.rawValue) — no \(names.joined(separator: "/")) set")
74 + continue
75 + }
76 + do {
77 + try store.setKey(key, for: provider)
78 + print(" stored \(provider.rawValue) (from \(name))")
79 + } catch {
80 + print(" FAILED \(provider.rawValue) — \(error.localizedDescription)")
81 + }
82 + }
83 + print("Vault: \(store.vaultURL.path)")
61 84 }
62 85
63 86 private static let usage = """
modified Sources/ZyquoAgent/Models/ToolTypes.swift +8 −1
@@ -37,11 +37,18 @@ struct ToolCall: Codable, Identifiable, Hashable {
37 37 /// Raw accumulated JSON string of the arguments; parsed and validated
38 38 /// against the ToolSpec schema by the agent loop before execution.
39 39 var argumentsJSON: String
40 + /// Gemini 3+ thought signature (`extra_content.google.thought_signature`
41 + /// on the compat endpoint). Opaque; captured from responses and echoed
42 + /// back verbatim when the call is threaded into history — Gemini rejects
43 + /// tool results whose originating call lost its signature (verified live,
44 + /// Phase 7.1). Nil for every other provider.
45 + var thoughtSignature: String?
40 46
41 − init(id: String, name: String, argumentsJSON: String) {
47 + init(id: String, name: String, argumentsJSON: String, thoughtSignature: String? = nil) {
42 48 self.id = id
43 49 self.name = name
44 50 self.argumentsJSON = argumentsJSON
51 + self.thoughtSignature = thoughtSignature
45 52 }
46 53
47 54 /// Arguments parsed to a dictionary; nil when the model produced
modified Sources/ZyquoAgent/Providers/OpenAICompatibleClient.swift +53 −4
@@ -112,16 +112,35 @@ struct OpenAICompatibleClient: ProviderClient {
112 112 }
113 113 }
114 114
115 − /// An assistant tool call echoed back into history.
115 + /// An assistant tool call echoed back into history. `extraContent` carries
116 + /// Gemini 3+ thought signatures (`extra_content.google.thought_signature`)
117 + /// back verbatim — Gemini rejects threaded tool results without them.
116 118 private struct WireToolCallOut: Encodable {
117 119 var id: String
118 120 var type = "function"
119 121 var function: Function
122 + var extraContent: ExtraContent?
123 +
124 + enum CodingKeys: String, CodingKey {
125 + case id, type, function
126 + case extraContent = "extra_content"
127 + }
120 128
121 129 struct Function: Encodable {
122 130 var name: String
123 131 var arguments: String
124 132 }
133 +
134 + struct ExtraContent: Codable {
135 + var google: Google
136 +
137 + struct Google: Codable {
138 + var thoughtSignature: String?
139 + enum CodingKeys: String, CodingKey {
140 + case thoughtSignature = "thought_signature"
141 + }
142 + }
143 + }
125 144 }
126 145
127 146 /// Message content: plain string, or an array of text/image parts for vision.
@@ -257,6 +276,14 @@ struct OpenAICompatibleClient: ProviderClient {
257 276 var index: Int?
258 277 var id: String?
259 278 var function: FunctionFragment?
279 + /// Gemini 3+ attaches `extra_content.google.thought_signature` to the
280 + /// call (both streaming fragments and non-streaming entries).
281 + var extraContent: WireToolCallOut.ExtraContent?
282 +
283 + enum CodingKeys: String, CodingKey {
284 + case index, id, function
285 + case extraContent = "extra_content"
286 + }
260 287
261 288 struct FunctionFragment: Decodable {
262 289 var name: String?
@@ -378,6 +405,15 @@ struct OpenAICompatibleClient: ProviderClient {
378 405 // Native tool calling (only for models that support it — providers
379 406 // reject `tools` on non-tool models).
380 407 if !request.tools.isEmpty, request.model.capabilities.tools {
408 + // OpenAI GPT-5.4+ rejects function tools combined with any
409 + // reasoning_effort other than "none" on /chat/completions
410 + // ("Function tools with reasoning_effort are not supported …
411 + // set reasoning_effort to 'none'" — verified live, Phase 7.1).
412 + // GPT-5.2 and earlier accept both together.
413 + if providerID == .openai, wire.reasoningEffort != nil,
414 + ["gpt-5.4", "gpt-5.5", "gpt-5.6"].contains(where: request.model.id.hasPrefix) {
415 + wire.reasoningEffort = "none"
416 + }
381 417 wire.tools = request.tools.map { spec in
382 418 WireTool(function: WireTool.Function(
383 419 name: spec.name,
@@ -420,7 +456,14 @@ struct OpenAICompatibleClient: ProviderClient {
420 456 calls.isEmpty ? nil : calls.map { call in
421 457 WireToolCallOut(
422 458 id: call.id,
423 − function: WireToolCallOut.Function(name: call.name, arguments: call.argumentsJSON)
459 + function: WireToolCallOut.Function(name: call.name, arguments: call.argumentsJSON),
460 + // Echo Gemini 3+ thought signatures back verbatim; other
461 + // providers never set one and never receive the field.
462 + extraContent: call.thoughtSignature.map {
463 + WireToolCallOut.ExtraContent(
464 + google: WireToolCallOut.ExtraContent.Google(thoughtSignature: $0)
465 + )
466 + }
424 467 )
425 468 }
426 469 }
@@ -446,6 +489,7 @@ struct OpenAICompatibleClient: ProviderClient {
446 489 var id: String?
447 490 var name: String?
448 491 var arguments = ""
492 + var thoughtSignature: String?
449 493 var announced = false
450 494 }
451 495
@@ -463,6 +507,9 @@ struct OpenAICompatibleClient: ProviderClient {
463 507 if let name = fragment.function?.name, !name.isEmpty {
464 508 partial.name = (partial.name ?? "") + name
465 509 }
510 + if let signature = fragment.extraContent?.google.thoughtSignature, !signature.isEmpty {
511 + partial.thoughtSignature = signature
512 + }
466 513 if !partial.announced, let name = partial.name {
467 514 partial.announced = true
468 515 if partial.id == nil {
@@ -486,7 +533,8 @@ struct OpenAICompatibleClient: ProviderClient {
486 533 ToolCall(
487 534 id: partial.id ?? "call_\(index)_\(UUID().uuidString.prefix(8))",
488 535 name: partial.name ?? "",
489 − argumentsJSON: partial.arguments.isEmpty ? "{}" : partial.arguments
536 + argumentsJSON: partial.arguments.isEmpty ? "{}" : partial.arguments,
537 + thoughtSignature: partial.thoughtSignature
490 538 )
491 539 }
492 540 }
@@ -583,7 +631,8 @@ struct OpenAICompatibleClient: ProviderClient {
583 631 ToolCall(
584 632 id: call.id ?? "call_\(call.index ?? offset)_\(UUID().uuidString.prefix(8))",
585 633 name: call.function?.name ?? "",
586 − argumentsJSON: call.function?.arguments ?? "{}"
634 + argumentsJSON: call.function?.arguments ?? "{}",
635 + thoughtSignature: call.extraContent?.google.thoughtSignature
587 636 )
588 637 }
589 638 }
added Sources/ZyquoAgent/Verify/VerifyHarness.swift +431 −0
@@ -0,0 +1,431 @@
1 +//
2 +// VerifyHarness.swift
3 +// Zyquo Agent
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Mail: contact@spboucher.ai
7 +//
8 +// Phase 7.1 — provider tool-calling verification, run against LIVE APIs with
9 +// the exact production provider clients. For every agent-capable model whose
10 +// provider has an API key available (environment or vault), the harness:
11 +//
12 +// 1. Streams a chat request offering ONE tool (`list_files`) and asserts the
13 +// model emits a tool call with the right name, syntactically valid JSON
14 +// arguments, and a normalized StopReason of `.toolUse`.
15 +// 2. Threads a canned tool_result back in the provider's wire format and
16 +// asserts the model produces a final text answer that mentions at least
17 +// one of the listed file names, with StopReason `.endTurn`.
18 +// 3. Asserts streaming actually streamed (more than one text/argument delta
19 +// across the exchange, not a single blob).
20 +//
21 +// Results are printed live and written as a markdown table to
22 +// docs/VERIFICATION.md. Providers run sequentially, models within a provider
23 +// sequentially, with a polite delay between calls. Keys are never logged.
24 +//
25 +// Usage: ZyquoAgent --verify [--provider <id>] [--model <id | provider/id>]
26 +//
27 +
28 +import Foundation
29 +
30 +enum VerifyHarness {
31 +
32 + // MARK: - Constants
33 +
34 + /// The single tool offered to every model. {path?: string} — an empty
35 + /// arguments object is a perfectly valid call.
36 + static let listFilesTool = ToolSpec(
37 + name: "list_files",
38 + description: "List the files in the current workspace directory.",
39 + parametersJSONSchema: """
40 + {"type":"object","properties":{"path":{"type":"string","description":"Optional subdirectory to list, relative to the workspace root."}}}
41 + """
42 + )
43 +
44 + static let systemPrompt = """
45 + You are Zyquo Agent, an autonomous agent operating inside a workspace directory on the user's Mac. \
46 + You must use the provided tools to inspect the workspace — never guess or fabricate file listings. \
47 + Call a tool when you need information. Once you have the tool result, answer the user in plain text \
48 + (naming the files you found) and stop; do not call the same tool again.
49 + """
50 +
51 + static let userPrompt = "List the files in the workspace using the tool."
52 +
53 + /// Canned tool_result threaded back on step 2.
54 + static let cannedListing = "MEMORY.md\nnotes.txt\nresult.csv"
55 + static let expectedNames = ["MEMORY.md", "notes.txt", "result.csv"]
56 +
57 + /// Whole-model budget (both steps + retries within a step).
58 + static let perModelTimeout: TimeInterval = 90
59 +
60 + // MARK: - Result model
61 +
62 + struct ModelReport {
63 + let model: AIModel
64 + var toolCall = false
65 + var threading = false
66 + var streaming = false
67 + var latency: TimeInterval?
68 + var notes: [String] = []
69 +
70 + var green: Bool { toolCall && threading && streaming }
71 + }
72 +
73 + private struct VerifyTimeout: Error {}
74 +
75 + /// Everything observed while one streamed response was consumed.
76 + private struct StreamOutcome {
77 + var text = ""
78 + var textDeltas = 0
79 + var reasoningDeltas = 0
80 + var toolCallStarts = 0
81 + var argDeltas = 0
82 + var toolCalls: [ToolCall] = []
83 + var rawReason: String?
84 + var stop: StopReason = .other(nil)
85 + }
86 +
87 + // MARK: - Entry
88 +
89 + static func run(arguments: [String]) async -> Int32 {
90 + let onlyProvider = value(after: "--provider", in: arguments).flatMap { ProviderID(rawValue: $0) }
91 + let onlyModel = value(after: "--model", in: arguments)
92 +
93 + var models = ModelCatalogData.all.filter(\.agentCapable)
94 + if let onlyProvider {
95 + models = models.filter { $0.provider == onlyProvider }
96 + }
97 + if let onlyModel {
98 + models = models.filter { $0.id == onlyModel || "\($0.provider.rawValue)/\($0.id)" == onlyModel }
99 + if models.isEmpty {
100 + FileHandle.standardError.write(Data("No agent-capable model matches “\(onlyModel)”.\n".utf8))
101 + return 64
102 + }
103 + }
104 +
105 + // Group by provider, in the canonical provider order.
106 + let providers = ProviderID.builtIn.filter { provider in
107 + models.contains { $0.provider == provider }
108 + }
109 + var keyless: [ProviderID] = []
110 + var reports: [ModelReport] = []
111 +
112 + let total = providers.reduce(0) { count, provider in
113 + AgentCLI.resolveAPIKey(for: provider) == nil
114 + ? count
115 + : count + models.filter { $0.provider == provider }.count
116 + }
117 + print("Zyquo Agent — provider tool-calling verification")
118 + print(" \(total) agent-capable model(s) across \(providers.count) provider(s)\n")
119 +
120 + for provider in providers {
121 + let providerModels = models.filter { $0.provider == provider }
122 + guard let key = AgentCLI.resolveAPIKey(for: provider) else {
123 + keyless.append(provider)
124 + print("⚠️ \(provider.displayName): no API key in environment or vault — \(providerModels.count) model(s) skipped")
125 + continue
126 + }
127 + let client = ProviderRegistry.client(for: provider)
128 + print("── \(provider.displayName) (\(providerModels.count) model(s)) " + String(repeating: "─", count: 30))
129 + for model in providerModels {
130 + var report = await checkWithRetry(model: model, client: client, key: key)
131 + if !report.green {
132 + report.notes.insert("failed twice (attempt 1 + retry)", at: 0)
133 + }
134 + reports.append(report)
135 + printLine(report)
136 + try? await Task.sleep(nanoseconds: interCallDelay(for: provider))
137 + }
138 + }
139 +
140 + writeReport(reports: reports, keyless: keyless)
141 +
142 + let green = reports.filter(\.green).count
143 + print("\n\(reports.count) model(s) tested · \(green) green · \(reports.count - green) failed")
144 + print("Full table: docs/VERIFICATION.md")
145 + return green == reports.count ? 0 : 1
146 + }
147 +
148 + // MARK: - Per-model check
149 +
150 + /// One attempt; on any failure, a single automatic retry (transient
151 + /// hiccups, rate-limit blips). Notes reflect the final attempt.
152 + private static func checkWithRetry(model: AIModel, client: any ProviderClient, key: String) async -> ModelReport {
153 + var start = Date()
154 + var first = await check(model: model, client: client, key: key)
155 + first.latency = Date().timeIntervalSince(start)
156 + if first.green { return first }
157 + try? await Task.sleep(nanoseconds: 2_000_000_000)
158 + start = Date()
159 + var second = await check(model: model, client: client, key: key)
160 + second.latency = Date().timeIntervalSince(start)
161 + if second.green {
162 + second.notes.append("passed on retry")
163 + }
164 + return second
165 + }
166 +
167 + private static func check(model: AIModel, client: any ProviderClient, key: String) async -> ModelReport {
168 + var report = ModelReport(model: model)
169 + let start = Date()
170 +
171 + var parameters = ChatParameters(maxTokens: 300)
172 + if model.parameterSupport.reasoningEffort { parameters.reasoningEffort = "low" }
173 + if model.parameterSupport.thinkingToggle { parameters.thinkingEnabled = false }
174 +
175 + // ---- Step 1: the model must call list_files -------------------------
176 + let step1 = ChatRequest(
177 + model: model,
178 + systemPrompt: systemPrompt,
179 + messages: [Message(role: .user, text: userPrompt)],
180 + parameters: parameters,
181 + tools: [listFilesTool],
182 + toolChoice: .auto
183 + )
184 + let outcome1: StreamOutcome
185 + do {
186 + outcome1 = try await collectPatiently(client, step1, apiKey: key, deadline: perModelTimeout)
187 + } catch {
188 + report.notes.append("step 1: \(describe(error))")
189 + return report
190 + }
191 +
192 + let calls = outcome1.toolCalls
193 + if calls.isEmpty {
194 + report.notes.append("step 1: no tool call emitted (stop=\(outcome1.rawReason ?? "nil"))"
195 + + (outcome1.text.isEmpty ? "" : "; text: “\(truncate(outcome1.text, 80))”"))
196 + if outcome1.stop == .maxTokens {
197 + report.notes.append("token-starved at 300 max_tokens")
198 + }
199 + return report
200 + }
201 + var callProblems: [String] = []
202 + if !calls.contains(where: { $0.name == listFilesTool.name }) {
203 + callProblems.append("wrong tool name “\(calls.map(\.name).joined(separator: ","))”")
204 + }
205 + for call in calls where call.argumentsDictionary == nil {
206 + callProblems.append("arguments not valid JSON object: \(truncate(call.argumentsJSON, 80))")
207 + }
208 + if outcome1.stop != .toolUse {
209 + callProblems.append("stop reason not toolUse (raw=\(outcome1.rawReason ?? "nil"))")
210 + }
211 + report.toolCall = callProblems.isEmpty
212 + report.notes.append(contentsOf: callProblems.map { "step 1: \($0)" })
213 + if calls.count > 1 {
214 + report.notes.append("emitted \(calls.count) parallel calls")
215 + }
216 + guard report.toolCall else { return report }
217 +
218 + // ---- Step 2: thread the canned tool_result back ----------------------
219 + let assistantTurn = Message.assistantToolCalls(calls, text: outcome1.text)
220 + let results = calls.map { ToolResult(toolCallID: $0.id, content: cannedListing) }
221 + let step2 = ChatRequest(
222 + model: model,
223 + systemPrompt: systemPrompt,
224 + messages: [
225 + Message(role: .user, text: userPrompt),
226 + assistantTurn,
227 + Message.toolResultsMessage(results),
228 + ],
229 + parameters: parameters,
230 + tools: [listFilesTool],
231 + toolChoice: .auto
232 + )
233 + let remaining = max(15, perModelTimeout - Date().timeIntervalSince(start))
234 + let outcome2: StreamOutcome
235 + do {
236 + outcome2 = try await collectPatiently(client, step2, apiKey: key, deadline: remaining)
237 + } catch {
238 + report.notes.append("step 2: \(describe(error))")
239 + return report
240 + }
241 +
242 + var threadProblems: [String] = []
243 + let answer = outcome2.text.trimmingCharacters(in: .whitespacesAndNewlines)
244 + if !outcome2.toolCalls.isEmpty {
245 + threadProblems.append("called the tool again after receiving the result")
246 + }
247 + if answer.isEmpty {
248 + threadProblems.append("empty final answer (stop=\(outcome2.rawReason ?? "nil"))")
249 + } else if !expectedNames.contains(where: { answer.localizedCaseInsensitiveContains($0) }) {
250 + threadProblems.append("final answer names none of the files: “\(truncate(answer, 80))”")
251 + }
252 + if outcome2.stop != .endTurn {
253 + threadProblems.append("stop reason not endTurn (raw=\(outcome2.rawReason ?? "nil"))")
254 + }
255 + report.threading = threadProblems.isEmpty
256 + report.notes.append(contentsOf: threadProblems.map { "step 2: \($0)" })
257 +
258 + // ---- Streaming: deltas must have arrived incrementally ---------------
259 + let deltas = outcome1.textDeltas + outcome1.argDeltas + outcome2.textDeltas
260 + report.streaming = deltas > 1
261 + if !report.streaming {
262 + report.notes.append("streaming: only \(deltas) delta event(s) — arrived as a single blob")
263 + }
264 + return report
265 + }
266 +
267 + // MARK: - Stream consumption
268 +
269 + /// `collect` with rate-limit patience: strict per-model RPM tiers and
270 + /// intermittent capacity waves (Moonshot's kimi-k3 429s with
271 + /// `engine_overloaded_error` on ~half of requests) clear after a short
272 + /// wait — retry up to twice with growing pauses. Other errors propagate.
273 + private static func collectPatiently(
274 + _ client: any ProviderClient,
275 + _ request: ChatRequest,
276 + apiKey: String,
277 + deadline: TimeInterval
278 + ) async throws -> StreamOutcome {
279 + var wait: TimeInterval = 20
280 + for _ in 0..<2 {
281 + do {
282 + return try await collect(client, request, apiKey: apiKey, deadline: deadline)
283 + } catch ProviderError.rateLimited(_, let retryAfter) {
284 + try await Task.sleep(nanoseconds: UInt64(max(retryAfter ?? wait, wait) * 1_000_000_000))
285 + wait *= 2
286 + }
287 + }
288 + return try await collect(client, request, apiKey: apiKey, deadline: deadline)
289 + }
290 +
291 + /// Consumes one streamed response fully, racing a deadline. Any thrown
292 + /// provider/network error propagates; a deadline hit throws VerifyTimeout.
293 + private static func collect(
294 + _ client: any ProviderClient,
295 + _ request: ChatRequest,
296 + apiKey: String,
297 + deadline: TimeInterval
298 + ) async throws -> StreamOutcome {
299 + try await withThrowingTaskGroup(of: StreamOutcome?.self) { group in
300 + group.addTask {
301 + var out = StreamOutcome()
302 + for try await event in client.streamChat(request, apiKey: apiKey) {
303 + switch event {
304 + case .textDelta(let delta):
305 + out.text += delta
306 + out.textDeltas += 1
307 + case .reasoningDelta:
308 + out.reasoningDeltas += 1
309 + case .toolCallStarted:
310 + out.toolCallStarts += 1
311 + case .toolCallArgumentsDelta:
312 + out.argDeltas += 1
313 + case .toolCalls(let calls):
314 + out.toolCalls = calls
315 + case .finished(let reason, let stop):
316 + out.rawReason = reason
317 + out.stop = stop
318 + case .citations, .usage:
319 + break
320 + }
321 + }
322 + return out
323 + }
324 + group.addTask {
325 + try await Task.sleep(nanoseconds: UInt64(deadline * 1_000_000_000))
326 + return nil
327 + }
328 + guard let first = try await group.next(), let outcome = first else {
329 + group.cancelAll()
330 + throw VerifyTimeout()
331 + }
332 + group.cancelAll()
333 + return outcome
334 + }
335 + }
336 +
337 + // MARK: - Reporting
338 +
339 + private static func printLine(_ report: ModelReport) {
340 + let marks = [
341 + report.toolCall ? "tool✅" : "tool❌",
342 + report.threading ? "thread✅" : "thread❌",
343 + report.streaming ? "stream✅" : "stream❌",
344 + ].joined(separator: " ")
345 + let latency = report.latency.map { String(format: "%.1fs", $0) } ?? "—"
346 + var line = " \(report.green ? "✅" : "❌") \(report.model.id) \(marks) \(latency)"
347 + if !report.notes.isEmpty {
348 + line += "\n · " + report.notes.joined(separator: "\n · ")
349 + }
350 + print(line)
351 + }
352 +
353 + private static func writeReport(reports: [ModelReport], keyless: [ProviderID]) {
354 + var lines: [String] = []
355 + lines.append("<!--")
356 + lines.append(" VERIFICATION.md")
357 + lines.append(" Zyquo Agent")
358 + lines.append(" Author: Simon-Pierre Boucher")
359 + lines.append(" Mail: contact@spboucher.ai")
360 + lines.append("-->")
361 + lines.append("")
362 + lines.append("# Provider Tool-Calling Verification — \(ISO8601DateFormatter().string(from: Date()))")
363 + lines.append("")
364 + lines.append("Phase 7.1 live sweep (`ZyquoAgent --verify`): every agent-capable model receives")
365 + lines.append("the `list_files` tool schema, must emit a valid tool call (streamed), consume the")
366 + lines.append("threaded `tool_result`, and produce a final answer naming the listed files.")
367 + lines.append("")
368 + let green = reports.filter(\.green).count
369 + lines.append("**\(reports.count) models tested · \(green) green · \(reports.count - green) failed**")
370 + if !keyless.isEmpty {
371 + lines.append("")
372 + lines.append("Providers skipped (no key available): \(keyless.map(\.displayName).joined(separator: ", "))")
373 + }
374 + lines.append("")
375 + lines.append("| Provider | Model | Tool call | Threading | Streaming | Latency | Notes |")
376 + lines.append("|---|---|---|---|---|---|---|")
377 + for report in reports {
378 + let latency = report.latency.map { String(format: "%.1fs", $0) } ?? "—"
379 + let notes = report.notes.joined(separator: "; ")
380 + .replacingOccurrences(of: "|", with: "\\|")
381 + .replacingOccurrences(of: "\n", with: " ")
382 + lines.append("| \(report.model.provider.displayName) | `\(report.model.id)` "
383 + + "| \(report.toolCall ? "✅" : "❌") | \(report.threading ? "✅" : "❌") "
384 + + "| \(report.streaming ? "✅" : "❌") | \(latency) | \(notes) |")
385 + }
386 + lines.append("")
387 + let url = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
388 + .appendingPathComponent("docs/VERIFICATION.md")
389 + try? lines.joined(separator: "\n").data(using: .utf8)?.write(to: url)
390 + }
391 +
392 + // MARK: - Helpers
393 +
394 + /// Gentle pacing between models of one provider (Cerebras free-tier rate
395 + /// limits are the strictest; everyone else gets a polite second).
396 + private static func interCallDelay(for provider: ProviderID) -> UInt64 {
397 + switch provider {
398 + case .cerebras: return 10_000_000_000
399 + case .mistral: return 2_000_000_000
400 + default: return 1_200_000_000
401 + }
402 + }
403 +
404 + private static func describe(_ error: Error) -> String {
405 + if error is VerifyTimeout { return "timed out after \(Int(perModelTimeout))s" }
406 + if let provider = error as? ProviderError {
407 + var text = provider.errorDescription ?? "\(provider)"
408 + if case .badRequest(_, let message) = provider {
409 + let lowered = (message ?? "").lowercased()
410 + if lowered.contains("model") &&
411 + (lowered.contains("not") || lowered.contains("exist") || lowered.contains("found")
412 + || lowered.contains("invalid") || lowered.contains("decommission")
413 + || lowered.contains("deprecat") || lowered.contains("access")) {
414 + text = "model id rejected by API — " + text
415 + }
416 + }
417 + return truncate(text, 220)
418 + }
419 + return truncate(error.localizedDescription, 220)
420 + }
421 +
422 + private static func truncate(_ text: String, _ limit: Int) -> String {
423 + let flat = text.replacingOccurrences(of: "\n", with: " ")
424 + return flat.count > limit ? String(flat.prefix(limit)) + "…" : flat
425 + }
426 +
427 + private static func value(after flag: String, in arguments: [String]) -> String? {
428 + guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else { return nil }
429 + return arguments[index + 1]
430 + }
431 +}
added docs/VERIFICATION.md +125 −0
@@ -0,0 +1,125 @@
1 +<!--
2 + VERIFICATION.md
3 + Zyquo Agent
4 +
5 + Author: Simon-Pierre Boucher
6 + Mail: contact@spboucher.ai
7 +-->
8 +
9 +# Provider Tool-Calling Verification — 2026-07-30
10 +
11 +Phase 7.1 live sweep (`ZyquoAgent --verify`): every agent-capable model receives
12 +the `list_files` tool schema, must emit a valid tool call (streamed), consume the
13 +threaded `tool_result`, and produce a final answer naming the listed files.
14 +
15 +**80 models tested · 77 green · 3 failed** (all 3 failures are provider-side —
16 +see the classification at the bottom)
17 +
18 +This table merges the full pass-1 sweep with the per-provider re-runs performed
19 +after the two client fixes below (each fixed model was re-verified green live):
20 +
21 +- **OpenAI GPT-5.4/5.5/5.6:** `/chat/completions` rejects function tools combined
22 + with any `reasoning_effort` other than `"none"` — the client now downgrades
23 + `reasoning_effort` to `"none"` whenever tools are attached for these models
24 + (`OpenAICompatibleClient.buildBody`).
25 +- **Gemini 3.x (compat endpoint):** tool calls carry an
26 + `extra_content.google.thought_signature` that MUST be echoed back verbatim when
27 + the call is threaded into history, or Gemini rejects the request
28 + (`INVALID_ARGUMENT: Function call is missing a thought_signature`). `ToolCall`
29 + now carries the opaque signature; the client captures it from streaming
30 + fragments / non-streaming entries and re-attaches it on assistant `tool_calls`.
31 +
32 +| Provider | Model | Tool call | Threading | Streaming | Latency | Notes |
33 +|---|---|---|---|---|---|---|
34 +| OpenAI | `gpt-5.6-sol` | ✅ | ✅ | ✅ | 3.0s | pass 1 ❌ (tools+reasoning_effort rejected); green after client fix |
35 +| OpenAI | `gpt-5.6-terra` | ✅ | ✅ | ✅ | 1.4s | pass 1 ❌ (tools+reasoning_effort rejected); green after client fix |
36 +| OpenAI | `gpt-5.6-luna` | ✅ | ✅ | ✅ | 1.6s | pass 1 ❌ (tools+reasoning_effort rejected); green after client fix |
37 +| OpenAI | `gpt-5.5` | ✅ | ✅ | ✅ | 4.1s | pass 1 ❌ (tools+reasoning_effort rejected); green after client fix |
38 +| OpenAI | `gpt-5.4` | ✅ | ✅ | ✅ | 1.8s | pass 1 ❌ (tools+reasoning_effort rejected); green after client fix |
39 +| OpenAI | `gpt-5.4-mini` | ✅ | ✅ | ✅ | 0.9s | pass 1 ❌ (tools+reasoning_effort rejected); green after client fix |
40 +| OpenAI | `gpt-5.2` | ✅ | ✅ | ✅ | 3.3s | |
41 +| OpenAI | `gpt-5.1` | ✅ | ✅ | ✅ | 1.6s | |
42 +| OpenAI | `gpt-5` | ✅ | ✅ | ✅ | 2.4s | |
43 +| OpenAI | `gpt-5-mini` | ✅ | ✅ | ✅ | 2.1s | |
44 +| OpenAI | `o3` | ✅ | ✅ | ✅ | 1.3s | |
45 +| OpenAI | `o4-mini` | ✅ | ✅ | ✅ | 1.2s | |
46 +| Anthropic | `claude-opus-5` | ✅ | ✅ | ✅ | 3.9s | |
47 +| Anthropic | `claude-sonnet-5` | ✅ | ✅ | ✅ | 4.0s | |
48 +| Anthropic | `claude-fable-5` | ✅ | ✅ | ✅ | 7.5s | |
49 +| Anthropic | `claude-opus-4-8` | ✅ | ✅ | ✅ | 3.2s | |
50 +| Anthropic | `claude-opus-4-7` | ✅ | ✅ | ✅ | 3.2s | |
51 +| Anthropic | `claude-opus-4-6` | ✅ | ✅ | ✅ | 6.1s | |
52 +| Anthropic | `claude-sonnet-4-6` | ✅ | ✅ | ✅ | 4.1s | |
53 +| Anthropic | `claude-haiku-4-5-20251001` | ✅ | ✅ | ✅ | 1.6s | |
54 +| xAI | `grok-4.5` | ✅ | ✅ | ✅ | 2.5s | |
55 +| xAI | `grok-4.3` | ✅ | ✅ | ✅ | 4.5s | |
56 +| xAI | `grok-4.20` | ✅ | ✅ | ✅ | 4.9s | |
57 +| xAI | `grok-4.20-non-reasoning` | ✅ | ✅ | ✅ | 1.2s | |
58 +| xAI | `grok-code-fast-1` | ✅ | ✅ | ✅ | 5.3s | |
59 +| Mistral | `mistral-medium-latest` | ✅ | ✅ | ✅ | 1.4s | |
60 +| Mistral | `mistral-large-latest` | ✅ | ✅ | ✅ | 3.7s | |
61 +| Mistral | `mistral-small-latest` | ✅ | ✅ | ✅ | 0.9s | |
62 +| Google Gemini | `gemini-3.6-flash` | ✅ | ✅ | ✅ | 2.1s | pass 1 ❌ (thought_signature missing on threading); green after client fix |
63 +| Google Gemini | `gemini-3.5-flash` | ✅ | ✅ | ✅ | 2.0s | pass 1 ❌ (thought_signature); green after client fix |
64 +| Google Gemini | `gemini-3.5-flash-lite` | ✅ | ✅ | ✅ | 1.0s | pass 1 ❌ (thought_signature); green after client fix |
65 +| Google Gemini | `gemini-3.1-pro-preview` | ✅ | ✅ | ✅ | 3.9s | pass 1 ❌ (thought_signature); green after client fix |
66 +| Google Gemini | `gemini-2.5-pro` | ✅ | ✅ | ✅ | 3.3s | |
67 +| Google Gemini | `gemini-2.5-flash` | ✅ | ✅ | ✅ | 1.3s | |
68 +| Google Gemini | `gemini-pro-latest` | ✅ | ✅ | ✅ | 4.2s | alias of Gemini 3.x; pass 1 ❌ (thought_signature); green after client fix |
69 +| Google Gemini | `gemini-flash-latest` | ✅ | ✅ | ✅ | 1.5s | alias of Gemini 3.x; pass 1 ❌ (thought_signature); green after client fix |
70 +| Alibaba Qwen | `qwen3.7-max` | ✅ | ✅ | ✅ | 3.0s | |
71 +| Alibaba Qwen | `qwen3.7-plus` | ✅ | ✅ | ✅ | 2.6s | |
72 +| Alibaba Qwen | `qwen3.7-flash` | ✅ | ✅ | ✅ | 1.6s | |
73 +| Alibaba Qwen | `qwen3.6-plus` | ✅ | ✅ | ✅ | 2.2s | |
74 +| Alibaba Qwen | `qwen3.5-plus` | ✅ | ✅ | ✅ | 2.0s | |
75 +| Alibaba Qwen | `qwen3-coder-plus` | ✅ | ✅ | ✅ | 2.1s | |
76 +| Alibaba Qwen | `qwen3-coder-flash` | ✅ | ✅ | ✅ | 2.0s | |
77 +| Alibaba Qwen | `qwen3-coder-next` | ✅ | ✅ | ✅ | 2.3s | |
78 +| Alibaba Qwen | `qwen3-coder-480b-a35b-instruct` | ✅ | ✅ | ✅ | 2.6s | |
79 +| Alibaba Qwen | `qwen3.5-397b-a17b` | ✅ | ✅ | ✅ | 1.9s | |
80 +| Alibaba Qwen | `deepseek-v4-pro` | ✅ | ✅ | ✅ | 4.6s | |
81 +| Alibaba Qwen | `glm-5.2` | ✅ | ✅ | ✅ | 2.1s | |
82 +| DeepSeek | `deepseek-v4-flash` | ✅ | ✅ | ✅ | 2.5s | |
83 +| DeepSeek | `deepseek-v4-pro` | ✅ | ✅ | ✅ | 3.2s | |
84 +| Kimi | `kimi-k3` | ❌ | ❌ | ❌ | 100.5s | INFRA: HTTP 429 `engine_overloaded_error` on most requests during the sweep window (reproduced with raw curl in every request shape; intermittent 200s prove the key/tier is fine). One harness attempt DID emit a correct `list_files` call, so the wire format is right — Moonshot capacity issue; re-verify when the engine recovers |
85 +| Kimi | `kimi-k2.7-code` | ✅ | ✅ | ✅ | 2.3s | |
86 +| Kimi | `kimi-k2.7-code-highspeed` | ✅ | ✅ | ✅ | 1.8s | |
87 +| Kimi | `kimi-k2.6` | ✅ | ✅ | ✅ | 5.1s | |
88 +| Kimi | `kimi-k2.5` | ✅ | ✅ | ✅ | 4.8s | |
89 +| Together AI | `moonshotai/Kimi-K3` | ✅ | ✅ | ✅ | 19.0s | slow first token (reasoning) |
90 +| Together AI | `moonshotai/Kimi-K2.7-Code` | ✅ | ✅ | ✅ | 1.6s | |
91 +| Together AI | `deepseek-ai/DeepSeek-V4-Pro` | ✅ | ✅ | ✅ | 1.6s | |
92 +| Together AI | `zai-org/GLM-5.2` | ✅ | ✅ | ✅ | 1.2s | |
93 +| Together AI | `Qwen/Qwen3.7-Max` | ✅ | ✅ | ✅ | 4.6s | |
94 +| Together AI | `openai/gpt-oss-120b` | ✅ | ✅ | ✅ | 2.2s | |
95 +| Together AI | `nvidia/nemotron-3-ultra-550b-a55b` | ✅ | ✅ | ✅ | 1.2s | |
96 +| Together AI | `MiniMaxAI/MiniMax-M3` | ✅ | ✅ | ✅ | 6.1s | |
97 +| DeepInfra | `anthropic/claude-fable-5` | ✅ | ✅ | ✅ | 6.3s | |
98 +| DeepInfra | `anthropic/claude-opus-5` | ✅ | ✅ | ✅ | 5.6s | |
99 +| DeepInfra | `anthropic/claude-sonnet-5` | ✅ | ✅ | ✅ | 7.1s | |
100 +| DeepInfra | `anthropic/claude-opus-4-8` | ✅ | ✅ | ✅ | 4.1s | |
101 +| DeepInfra | `anthropic/claude-haiku-4-5` | ✅ | ✅ | ✅ | 2.3s | |
102 +| DeepInfra | `google/gemini-3.1-pro` | ✅ | ❌ | ❌ | 4.0s | SERVER-SIDE: step 1 works, but DeepInfra never returns Gemini's `thought_signature` and strips `extra_content` from requests (verified with raw probes, incl. Google's documented bypass placeholder) → Google rejects every threaded tool result with INVALID_ARGUMENT. Multi-turn tool use is impossible through DeepInfra — REMOVAL CANDIDATE (the same model is green via the native Gemini provider) |
103 +| DeepInfra | `google/gemini-3.5-flash` | ✅ | ❌ | ❌ | 2.3s | SERVER-SIDE: same as `google/gemini-3.1-pro` — thought_signature dropped by DeepInfra; REMOVAL CANDIDATE |
104 +| DeepInfra | `deepseek-ai/DeepSeek-V4-Pro` | ✅ | ✅ | ✅ | 4.1s | |
105 +| DeepInfra | `deepseek-ai/DeepSeek-V4-Flash` | ✅ | ✅ | ✅ | 3.3s | |
106 +| DeepInfra | `moonshotai/Kimi-K2.7-Code` | ✅ | ✅ | ✅ | 2.2s | |
107 +| DeepInfra | `zai-org/GLM-5.2` | ✅ | ✅ | ✅ | 3.1s | |
108 +| DeepInfra | `Qwen/Qwen3.7-Max` | ✅ | ✅ | ✅ | 4.7s | |
109 +| DeepInfra | `Qwen/Qwen3-Coder-480B-A35B-Instruct-Turbo` | ✅ | ✅ | ✅ | 0.8s | |
110 +| DeepInfra | `openai/gpt-oss-120b` | ✅ | ✅ | ✅ | 2.6s | |
111 +| DeepInfra | `MiniMaxAI/MiniMax-M3` | ✅ | ✅ | ✅ | 3.0s | |
112 +| Cerebras | `gpt-oss-120b` | ✅ | ✅ | ✅ | 1.4s | |
113 +| Cerebras | `gemma-4-31b` | ✅ | ✅ | ✅ | 0.6s | |
114 +
115 +## Failure classification
116 +
117 +| Model | Class | Evidence / disposition |
118 +|---|---|---|
119 +| `deepinfra` `google/gemini-3.1-pro` | Broken server-side | DeepInfra never emits `thought_signature` and drops `extra_content` on requests; Gemini 3 requires it for threaded tool calls. No client-side workaround exists (Google's documented placeholder is also stripped). **Recommend removing from the agent-capable set** — the identical models are green via the native Gemini provider. |
120 +| `deepinfra` `google/gemini-3.5-flash` | Broken server-side | Same as above. **Recommend removing from the agent-capable set.** |
121 +| `kimi` `kimi-k3` | Infra (provider capacity) | HTTP 429 `engine_overloaded_error` on most requests during the sweep window, reproduced with raw curl in every request shape; intermittent 200s prove the key/tier is fine and one harness attempt did emit a correct `list_files` call. Not a wire-format or capability failure — keep in the set, re-verify when Moonshot capacity recovers. |
122 +
123 +No model id in the agent-capable set was rejected as nonexistent/deprecated, and
124 +no model failed tool calling because of model capability — every ❌ above is
125 +provider-side.
126