spb/zyquo-atlas Public License

1//2//  VerifyHarness.swift3//  Zyquo Atlas4//5//  Author: Simon-Pierre Boucher6//  Mail: contact@spboucher.ai7//8//  Phase 7 verification. Exercises the *Atlas* stack against live APIs with real9//  keys: (1) content extraction on real pages, (2) a grounded summarize across10//  every catalog model through the ported provider clients, (3) the core AI11//  action matrix, and (4) cancel-on-navigation. Results → stdout + docs/12//  VERIFICATION.md. Keys come from the environment (source .env.keys) and are13//  never logged or persisted. `--load-vault` seeds the vault; `--quick` runs one14//  model per provider.15//1617import Foundation18import WebKit19import AppKit2021@MainActor22enum VerifyHarness {23    static let environmentKeys: [ProviderID: String] = [24        .openai: "OPENAI_API_KEY", .anthropic: "ANTHROPIC_API_KEY", .xai: "XAI_API_KEY",25        .mistral: "MISTRAL_API_KEY", .gemini: "GEMINI_API_KEY", .qwen: "DASHSCOPE_API_KEY",26        .deepseek: "DEEPSEEK_API_KEY", .kimi: "MOONSHOT_API_KEY", .perplexity: "PERPLEXITY_API_KEY",27        .together: "TOGETHER_API_KEY", .deepinfra: "DEEPINFRA_API_KEY", .cerebras: "CEREBRAS_API_KEY",28    ]2930    struct Result {31        let section: String32        let provider: String33        let subject: String34        let test: String35        let passed: Bool36        let latency: TimeInterval?37        let detail: String38    }3940    // MARK: - Entry4142    static func run(arguments: [String]) async -> Int32 {43        let quick = arguments.contains("--quick")44        let onlyProvider = value(after: "--provider", in: arguments).flatMap { ProviderID(rawValue: $0) }45        let catalog = ModelCatalog()4647        print("Zyquo Atlas — Phase 7 verification\(quick ? " (quick)" : "")\n")48        var results: [Result] = []4950        // 1) Extraction-quality suite (headless real pages).51        let (extraction, sample) = await verifyExtraction()52        results += extraction5354        // A grounded PageContext for the model sweep + action matrix. Trim so55        // every one of 170 model calls stays cheap and fast while still grounded.56        let context = trimmed(sample) ?? fallbackContext()5758        // 2) Grounded summarize across every catalog model.59        results += await verifyModels(catalog: catalog, context: context,60                                      quick: quick, onlyProvider: onlyProvider)6162        // 3) Core AI action matrix (against one live model per available provider).63        results += await verifyActions(catalog: catalog, context: context, onlyProvider: onlyProvider)6465        // 4) Cancellation + privacy.66        results += await verifyCancellation(catalog: catalog, context: context)6768        report(results)69        return results.contains { !$0.passed } ? 1 : 070    }7172    // MARK: - 1. Extraction suite7374    private static let extractionPages: [(name: String, url: String, selectFirstPara: Bool)] = [75        ("article",      "https://en.wikipedia.org/wiki/Cartography", false),76        ("docs",         "https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch", false),77        ("long-article", "https://en.wikipedia.org/wiki/World_War_II", false),78        ("js-app",       "https://react.dev/", false),79        ("selection",    "https://en.wikipedia.org/wiki/Map", true),80    ]8182    private static func verifyExtraction() async -> ([Result], PageContext?) {83        print("— Content extraction —")84        var results: [Result] = []85        var sample: PageContext?86        let config = ProfileStore.shared.makeConfiguration(for: .defaultProfile)87        let loader = PageLoader(configuration: config)8889        for page in extractionPages {90            guard let url = URL(string: page.url) else { continue }91            let start = Date()92            do {93                try await loader.load(url)94                try? await Task.sleep(nanoseconds: 1_500_000_000)95                if page.selectFirstPara {96                    // Select the first real paragraph so the extractor captures it.97                    _ = try? await loader.webView.evaluateJavaScript(98                        "(function(){var p=Array.from(document.querySelectorAll('p')).find(function(e){return e.textContent.trim().length>80});if(!p)return false;var r=document.createRange();r.selectNodeContents(p);var s=getSelection();s.removeAllRanges();s.addRange(r);return true;})();",99                        in: nil, contentWorld: .page)100                    try? await Task.sleep(nanoseconds: 400_000_000)101                }102                let ctx = try await ContentExtractor.extract(from: loader.webView)103                let ok: Bool104                let detail: String105                if page.selectFirstPara {106                    ok = (ctx.selection?.text.isEmpty == false)107                    detail = "selection=\(ctx.selection?.text.prefix(40).description ?? "nil"), words=\(ctx.wordCount)"108                } else {109                    ok = ctx.wordCount > 80 && !ctx.markdown.isEmpty110                    detail = "quality=\(ctx.quality.rawValue), words=\(ctx.wordCount), ~\(ctx.estimatedTokens)tok, headings=\(ctx.headings.count)"111                }112                results.append(Result(section: "Extraction", provider: "—", subject: page.name,113                                      test: "extract", passed: ok,114                                      latency: Date().timeIntervalSince(start), detail: detail))115                if page.name == "article" { sample = ctx }116                print("  \(ok ? "✓" : "✗") \(page.name): \(detail)")117            } catch {118                results.append(Result(section: "Extraction", provider: "—", subject: page.name,119                                      test: "extract", passed: false, latency: nil,120                                      detail: error.localizedDescription))121                print("  ✗ \(page.name): \(error.localizedDescription)")122            }123        }124        return (results, sample)125    }126127    // MARK: - 2. Model sweep (grounded summarize on every model)128129    private static func verifyModels(catalog: ModelCatalog, context: PageContext,130                                     quick: Bool, onlyProvider: ProviderID?) async -> [Result] {131        print("\n— AI summarize across all catalog models —")132        var results: [Result] = []133        let providers = ProviderID.builtIn.filter { onlyProvider == nil || $0 == onlyProvider }134135        for provider in providers {136            guard let key = apiKey(for: provider) else {137                print("  ⚠︎ \(provider.rawValue): no key — skipped"); continue138            }139            var models = catalog.models(for: provider)140            if quick { models = Array(models.prefix(1)) }141            let serial = provider == .cerebras || provider == .mistral142            if serial {143                for model in models {144                    results.append(await summarizeTest(model: model, context: context, key: key))145                    // Cerebras free tier: 5 req/min.146                    try? await Task.sleep(nanoseconds: provider == .cerebras ? 13_000_000_000 : 1_100_000_000)147                }148            } else {149                results += await limitedConcurrent(models, limit: 3) { model in150                    await summarizeTest(model: model, context: context, key: key)151                }152            }153            let pass = results.filter { $0.section == "Models" && $0.provider == provider.displayName && $0.passed }.count154            let total = results.filter { $0.section == "Models" && $0.provider == provider.displayName }.count155            print("  \(pass == total ? "✓" : "✗") \(provider.displayName): \(pass)/\(total) models")156        }157        return results158    }159160    private static func summarizeTest(model: AIModel, context: PageContext, key: String) async -> Result {161        let start = Date()162        let client = ProviderRegistry.client(for: model)163        // Reasoning models stream hidden thinking first; give them room to reach text.164        var params = ChatParameters(maxTokens: model.capabilities.reasoning ? 8000 : 256)165        if model.parameterSupport.reasoningEffort { params.reasoningEffort = "low" }166        let request = ChatRequest(167            model: model, systemPrompt: AIAction.summarize.systemGuidance,168            messages: [Message(role: .user, text: AIAction.summarize.userPrompt(for: context))],169            parameters: params)170        do {171            var deltas = 0, text = ""172            for try await event in client.streamChat(request, apiKey: key) {173                if case .textDelta(let d) = event { deltas += 1; text += d }174            }175            let ok = deltas >= 1 && !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty176            return Result(section: "Models", provider: model.provider.displayName, subject: model.id,177                          test: "summarize", passed: ok, latency: Date().timeIntervalSince(start),178                          detail: ok ? "deltas=\(deltas)" : "empty response")179        } catch {180            return Result(section: "Models", provider: model.provider.displayName, subject: model.id,181                          test: "summarize", passed: false, latency: Date().timeIntervalSince(start),182                          detail: error.localizedDescription)183        }184    }185186    // MARK: - 3. Action matrix187188    private static func verifyActions(catalog: ModelCatalog, context: PageContext,189                                      onlyProvider: ProviderID?) async -> [Result] {190        print("\n— Core AI action matrix —")191        // Pick the first provider with a key (or the requested one).192        let providers = ProviderID.builtIn.filter { onlyProvider == nil || $0 == onlyProvider }193        guard let provider = providers.first(where: { apiKey(for: $0) != nil }),194              let model = catalog.cheapestModel(for: provider) ?? catalog.models(for: provider).first,195              let key = apiKey(for: provider) else {196            print("  ⚠︎ no keyed provider for action matrix"); return []197        }198199        var results: [Result] = []200        func record(_ name: String, _ system: String?, _ user: String) async {201            let start = Date()202            let client = ProviderRegistry.client(for: model)203            var p = ChatParameters(maxTokens: 256)204            if model.parameterSupport.reasoningEffort { p.reasoningEffort = "low" }205            let req = ChatRequest(model: model, systemPrompt: system,206                                  messages: [Message(role: .user, text: user)], parameters: p)207            do {208                var text = ""209                for try await ev in client.streamChat(req, apiKey: key) {210                    if case .textDelta(let d) = ev { text += d }211                }212                let ok = !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty213                results.append(Result(section: "Actions", provider: model.provider.displayName,214                                      subject: name, test: "action", passed: ok,215                                      latency: Date().timeIntervalSince(start),216                                      detail: ok ? "" : "empty"))217                print("  \(ok ? "✓" : "✗") \(name)")218            } catch {219                results.append(Result(section: "Actions", provider: model.provider.displayName,220                                      subject: name, test: "action", passed: false, latency: nil,221                                      detail: error.localizedDescription))222                print("  ✗ \(name): \(error.localizedDescription)")223            }224        }225226        let selText = "Cartography is the study and practice of making and using maps."227        await record("omnibox-ask", "You answer grounded in the page.",228                     AIAction.ask.userPrompt(for: context, extra: "What is this page about?"))229        await record("summarize", AIAction.summarize.systemGuidance, AIAction.summarize.userPrompt(for: context))230        await record("chat-followup", AIAction.ask.systemGuidance,231                     AIAction.ask.userPrompt(for: context, extra: "List two key facts."))232        await record("selection-explain", AIAction.explainSelection.selectionPrompt(selText).system,233                     AIAction.explainSelection.selectionPrompt(selText).user)234        await record("selection-translate", AIAction.translate.selectionPrompt(selText, language: "French").system,235                     AIAction.translate.selectionPrompt(selText, language: "French").user)236        await record("selection-rewrite", AIAction.rewrite.selectionPrompt(selText).system,237                     AIAction.rewrite.selectionPrompt(selText).user)238        await record("multi-tab-compare", "You compare multiple web pages.",239                     "Compare these two pages briefly:\n\nA) \(context.title)\n\(context.markdown.prefix(800))\n\nB) A blank page.")240        return results241    }242243    // MARK: - 4. Cancellation + privacy244245    private static func verifyCancellation(catalog: ModelCatalog, context: PageContext) async -> [Result] {246        print("\n— Cancellation & privacy —")247        guard let provider = ProviderID.builtIn.first(where: { apiKey(for: $0) != nil }),248              let model = catalog.cheapestModel(for: provider) ?? catalog.models(for: provider).first,249              let key = apiKey(for: provider) else { return [] }250251        let service = AIService()252        service.run(.summarize, on: context, model: model, apiKey: key)253        try? await Task.sleep(nanoseconds: 300_000_000)254        service.cancel()255        try? await Task.sleep(nanoseconds: 400_000_000)256        let stopped = !service.isStreaming257        print("  \(stopped ? "✓" : "✗") cancel-on-navigation stops the stream")258259        // Privacy invariant: a fresh service sends nothing until run() is called.260        let idle = AIService()261        let noSend = idle.output.isEmpty && !idle.isStreaming262        print("  \(noSend ? "✓" : "✗") no content leaves the device without a user action")263264        return [265            Result(section: "Privacy", provider: "—", subject: "cancel-on-navigation", test: "cancel",266                   passed: stopped, latency: nil, detail: stopped ? "stream cancelled" : "still streaming"),267            Result(section: "Privacy", provider: "—", subject: "no-send-without-action", test: "privacy",268                   passed: noSend, latency: nil, detail: "AIService idle until run()"),269        ]270    }271272    // MARK: - Reporting273274    private static func report(_ results: [Result]) {275        let fails = results.filter { !$0.passed }276        var lines = ["<!--", "  VERIFICATION.md", "  Zyquo Atlas", "",277                     "  Author: Simon-Pierre Boucher", "  Mail: contact@spboucher.ai", "-->", "",278                     "# Zyquo Atlas — Phase 7 Verification",279                     "",280                     "**\(results.count) checks · \(results.count - fails.count) passed · \(fails.count) failed**",281                     "",282                     "Exercises the Atlas stack (ContentExtractor + AIService + ported provider clients)",283                     "against live APIs with real keys. Page content is sent only on an AI action; keys",284                     "come from the environment and are never logged.", ""]285286        func table(_ section: String, cols: [String]) {287            let rows = results.filter { $0.section == section }288            guard !rows.isEmpty else { return }289            lines.append("## \(section)")290            lines.append("")291            lines.append("| " + cols.joined(separator: " | ") + " |")292            lines.append("|" + cols.map { _ in "---" }.joined(separator: "|") + "|")293            for r in rows.sorted(by: { ($0.provider, $0.subject) < ($1.provider, $1.subject) }) {294                let lat = r.latency.map { String(format: "%.1fs", $0) } ?? "—"295                let d = r.detail.replacingOccurrences(of: "|", with: "\\|")296                lines.append("| \(r.provider) | `\(r.subject)` | \(r.test) | \(r.passed ? "✅" : "❌") | \(lat) | \(d) |")297            }298            lines.append("")299        }300        table("Extraction", cols: ["Source", "Page", "Test", "Result", "Latency", "Detail"])301        table("Models", cols: ["Provider", "Model", "Test", "Result", "Latency", "Detail"])302        table("Actions", cols: ["Provider", "Action", "Test", "Result", "Latency", "Detail"])303        table("Privacy", cols: ["—", "Check", "Test", "Result", "Latency", "Detail"])304305        let doc = lines.joined(separator: "\n")306        let url = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)307            .appendingPathComponent("docs/VERIFICATION.md")308        try? doc.data(using: .utf8)?.write(to: url)309310        print("\n" + String(repeating: "—", count: 64))311        for f in fails { print("❌ [\(f.section)] \(f.provider) \(f.subject) [\(f.test)] — \(f.detail)") }312        print(String(repeating: "—", count: 64))313        print("\(results.count) checks · \(results.count - fails.count) passed · \(fails.count) failed")314        print("Full table: docs/VERIFICATION.md")315    }316317    // MARK: - Vault seeding318319    static func loadVault() {320        let store = SecureKeyStore()321        var loaded: [String] = []322        for (provider, name) in environmentKeys {323            if let key = ProcessInfo.processInfo.environment[name], !key.isEmpty {324                try? store.setKey(key, for: provider)325                loaded.append(provider.rawValue)326            }327        }328        print("Vault updated with keys for: \(loaded.sorted().joined(separator: ", "))")329    }330331    // MARK: - Helpers332333    static func apiKey(for provider: ProviderID) -> String? {334        guard let name = environmentKeys[provider] else { return nil }335        let value = ProcessInfo.processInfo.environment[name]336        return (value?.isEmpty ?? true) ? nil : value337    }338339    private static func value(after flag: String, in args: [String]) -> String? {340        guard let i = args.firstIndex(of: flag), i + 1 < args.count else { return nil }341        return args[i + 1]342    }343344    /// Trims a PageContext's markdown so 170 model calls stay cheap but grounded.345    private static func trimmed(_ ctx: PageContext?) -> PageContext? {346        guard let ctx, !ctx.markdown.isEmpty else { return nil }347        let short = String(ctx.markdown.prefix(1800))348        return PageContext(cloning: ctx, markdown: short)349    }350351    /// A baked context if extraction failed entirely (keeps the sweep runnable).352    private static func fallbackContext() -> PageContext {353        PageContext(bakedTitle: "Cartography",354                    url: "https://en.wikipedia.org/wiki/Cartography",355                    markdown: "# Cartography\n\nCartography is the study and practice of making and using maps. Combining science, aesthetics and technique, it builds on the premise that reality can be modeled in ways that communicate spatial information effectively.")356    }357358    private static func limitedConcurrent<T: Sendable>(359        _ items: [AIModel], limit: Int,360        _ op: @escaping @Sendable (AIModel) async -> T361    ) async -> [T] {362        await withTaskGroup(of: (Int, T).self) { group in363            var out: [(Int, T)] = []364            var it = items.enumerated().makeIterator()365            var inFlight = 0366            func addNext() { if let (i, m) = it.next() { inFlight += 1; group.addTask { (i, await op(m)) } } }367            for _ in 0..<limit { addNext() }368            while inFlight > 0 {369                if let r = await group.next() { out.append(r); inFlight -= 1; addNext() }370            }371            return out.sorted { $0.0 < $1.0 }.map { $0.1 }372        }373    }374}375