spb/zyquo-local Public MIT

1//2//  VerifyRunner.swift3//  Zyquo Local4//5//  Author: Simon-Pierre Boucher6//  Mail: contact@spboucher.ai7//89import Foundation1011/// Phase 7 verification harness: `ZyquoLocal --verify [--keep]`.12/// Proves downloading and running real models works end-to-end on this Mac,13/// and dry-verifies the entire Featured catalog against the live Hub.14enum VerifyRunner {15    /// Models under test: span architectures and sizes that fit this Mac.16    private static let testRepos = [17        "mlx-community/Qwen3-0.6B-4bit",                 // tiny, qwen318        "mlx-community/Llama-3.2-1B-Instruct-4bit",      // tiny, llama19        "mlx-community/SmolLM3-3B-4bit",                 // 3B, smollm320        "mlx-community/gemma-3-4b-it-qat-4bit",          // 4B, gemma321        "mlx-community/Qwen2.5-Coder-7B-Instruct-4bit",  // coding, qwen222        "mlx-community/DeepSeek-R1-0528-Qwen3-8B-4bit",  // reasoning distill23    ]2425    struct RunResult {26        var repoID: String27        var download = "–"28        var load = "–"29        var generate = "–"30        var multiTurn = "–"31        var cancel = "–"32        var unload = "–"33        var tokensPerSecond: Double?34        var ttft: TimeInterval?35        var note = ""36    }3738    @MainActor39    static func run(arguments: [String]) async {40        let keepAll = arguments.contains("--keep")41        var results: [RunResult] = []4243        let store = ModelStore()44        let hub = HubService(token: nil)45        let downloads = DownloadManager(hub: hub, store: store)4647        for repoID in testRepos {48            var result = RunResult(repoID: repoID)49            log("\n═══ \(repoID) ═══")5051            // 1 — download (or reuse)52            if store.model(for: repoID) == nil {53                log("downloading…")54                await downloads.download(repoID: repoID)55                var done = false56                var lastPercent = -157                while !done {58                    try? await Task.sleep(for: .milliseconds(500))59                    guard let t = downloads.task(for: repoID) else {60                        result.note = "download task vanished"61                        break62                    }63                    switch t.state {64                    case .completed:65                        done = true66                    case .failed:67                        result.note = t.errorDescription ?? "download failed"68                        done = true69                    default:70                        let percent = Int(t.fractionCompleted * 100)71                        if percent / 10 != lastPercent / 10 {72                            lastPercent = percent73                            log("  \(percent)%  \(downloads.speeds[repoID].map(formatSpeed) ?? "")")74                        }75                    }76                }77            }78            store.rescan()79            guard let model = store.model(for: repoID) else {80                result.download = "❌"81                results.append(result)82                log("  download ❌ \(result.note)")83                continue84            }85            guard ModelStore.isValidModelDirectory(model.directory) else {86                result.download = "❌"87                result.note = "invalid directory after download"88                results.append(result)89                continue90            }91            result.download = "✅"92            log("  download ✅ (\(formatBytes(model.sizeBytes)), arch=\(model.architecture ?? "?"))")9394            // 2 — load95            let engine = InferenceEngine()96            do {97                try await engine.load(model: model)98                result.load = "✅"99                log("  load ✅")100            } catch {101                result.load = "❌"102                result.note = error.localizedDescription103                results.append(result)104                log("  load ❌ \(error.localizedDescription)")105                continue106            }107108            // 3 — deterministic generation109            do {110                let conversation = Conversation(111                    params: GenerationParams(temperature: 0, maxTokens: 600))112                try await engine.startSession(conversation: conversation)113                let (text, stats) = try await collect(114                    engine: engine, prompt: "Reply with exactly: OK",115                    params: conversation.params)116                var parser = ThinkTagParser()117                let (visible, _, _) = parser.consume(text)118                let cleaned = visible.trimmingCharacters(in: .whitespacesAndNewlines)119                if cleaned.isEmpty {120                    result.generate = "❌"121                    result.note = "empty output"122                } else {123                    result.generate = "✅"124                    result.ttft = stats?.timeToFirstToken125                    log("  generate ✅ “\(String(cleaned.prefix(60)))”")126                }127            } catch {128                result.generate = "❌"129                result.note = error.localizedDescription130                log("  generate ❌ \(error.localizedDescription)")131            }132133            // 4 — multi-turn context carry-over134            do {135                let conversation = Conversation(136                    params: GenerationParams(temperature: 0, maxTokens: 800))137                try await engine.startSession(conversation: conversation)138                _ = try await collect(139                    engine: engine,140                    prompt: "My favorite color is vermilion. Just say: noted.",141                    params: conversation.params)142                let (answer, stats) = try await collect(143                    engine: engine, prompt: "What is my favorite color? Answer in one word.",144                    params: conversation.params)145                if answer.localizedCaseInsensitiveContains("vermilion") {146                    result.multiTurn = "✅"147                    result.tokensPerSecond = stats?.tokensPerSecond148                    log("  multi-turn ✅")149                } else {150                    result.multiTurn = "❌"151                    result.note = "no context carry-over: “\(String(answer.suffix(80)))”"152                    log("  multi-turn ❌ \(result.note)")153                }154                if result.tokensPerSecond == nil { result.tokensPerSecond = stats?.tokensPerSecond }155            } catch {156                result.multiTurn = "❌"157                result.note = error.localizedDescription158            }159160            // 5 — streaming cancellation161            do {162                let conversation = Conversation(params: GenerationParams(temperature: 0.7))163                try await engine.startSession(conversation: conversation)164                let stream = try await engine.generate(165                    prompt: "Write a very long story about the ocean.",166                    params: conversation.params)167                var tokens = 0168                let start = Date()169                var sawEnd = false170                let consumer = Task {171                    for try await event in stream {172                        if case .token = event {173                            tokens += 1174                            if tokens == 12 { break }  // cancels via onTermination175                        }176                    }177                }178                _ = try? await consumer.value179                await engine.stopGeneration()180                sawEnd = true181                let elapsed = Date().timeIntervalSince(start)182                if sawEnd && elapsed < 30 {183                    result.cancel = "✅"184                    log("  cancel ✅ (stopped after \(tokens) tokens, \(String(format: "%.1f", elapsed))s)")185                } else {186                    result.cancel = "❌"187                }188            } catch {189                result.cancel = "❌"190                result.note = error.localizedDescription191            }192193            // 6 — unload + memory release194            let before = MemoryAdvisor.activeMemoryBytes195            await engine.unload()196            try? await Task.sleep(for: .milliseconds(500))197            let after = MemoryAdvisor.activeMemoryBytes198            if after < max(200_000_000, before / 4) {199                result.unload = "✅"200                log("  unload ✅ (\(formatBytes(Int64(before))) → \(formatBytes(Int64(after))))")201            } else {202                result.unload = "❌"203                result.note += " memory not released (\(formatBytes(Int64(after))))"204                log("  unload ❌ (\(formatBytes(Int64(before))) → \(formatBytes(Int64(after))))")205            }206207            results.append(result)208        }209210        // Reclaim disk: keep the smallest model for ongoing dev.211        if !keepAll {212            for repoID in testRepos.dropFirst() where store.model(for: repoID) != nil {213                store.delete(repoID: repoID)214                log("deleted \(repoID) to reclaim disk")215            }216        }217218        // ── Catalog dry-verification ────────────────────────────────────────219        log("\n═══ Featured catalog dry-verification (30 repos) ═══")220        var catalogRows: [(String, String, String)] = []221        for entry in ModelCatalog.featured {222            do {223                let (files, total) = try await hub.requiredFiles(of: entry.repoID)224                let gb = Double(total) / 1_000_000_000225                let deviation = abs(gb - entry.sizeGB) / entry.sizeGB226                let sizeOK = deviation < 0.10227                catalogRows.append((228                    entry.repoID,229                    "✅ \(files.count) files",230                    sizeOK231                        ? String(format: "✅ %.2f GB", gb)232                        : String(format: "⚠️ %.2f GB (catalog says %.2f)", gb, entry.sizeGB)233                ))234            } catch {235                catalogRows.append((entry.repoID, "❌ \(error.localizedDescription)", "–"))236            }237        }238        for row in catalogRows {239            log("  \(row.0): \(row.1) · \(row.2)")240        }241242        // ── Results table ───────────────────────────────────────────────────243        var table = """244            | Model | Download | Load | Generate | Multi-turn | Cancel | Unload | tok/s | TTFT |245            |---|---|---|---|---|---|---|---|---|246247            """248        for r in results {249            table += "| \(shortModelName(r.repoID)) | \(r.download) | \(r.load) | \(r.generate) | \(r.multiTurn) | \(r.cancel) | \(r.unload) | \(r.tokensPerSecond.map { String(format: "%.1f", $0) } ?? "–") | \(r.ttft.map { String(format: "%.2fs", $0) } ?? "–") |\n"250        }251        log("\n" + table)252253        let allGreen = results.allSatisfy {254            $0.download == "✅" && $0.load == "✅" && $0.generate == "✅"255                && $0.multiTurn == "✅" && $0.cancel == "✅" && $0.unload == "✅"256        }257        let catalogGreen = catalogRows.allSatisfy { $0.1.hasPrefix("✅") }258        log(allGreen && catalogGreen ? "\nVERIFY: ALL GREEN" : "\nVERIFY: FAILURES PRESENT")259        for r in results where !r.note.isEmpty {260            log("  note[\(shortModelName(r.repoID))]: \(r.note)")261        }262        exit(allGreen && catalogGreen ? 0 : 1)263    }264265    /// Collects one full generation, returning the raw text + final stats.266    @MainActor267    private static func collect(268        engine: InferenceEngine, prompt: String, params: GenerationParams269    ) async throws -> (String, GenerationStats?) {270        var text = ""271        var stats: GenerationStats?272        let stream = try await engine.generate(prompt: prompt, params: params)273        for try await event in stream {274            switch event {275            case .token(let t): text += t276            case .stats(let s): stats = s277            case .finished: break278            }279        }280        return (text, stats)281    }282283    private static func log(_ text: String) {284        FileHandle.standardError.write(Data((text + "\n").utf8))285    }286}287