//
//  VerifyHarness.swift
//  Zyquo Cloud
//
//  Author: Simon-Pierre Boucher
//  Mail: contact@spboucher.ai
//
//  Phase 7 API verification: exercises the exact production provider clients
//  against live APIs with real keys. Run via `ZyquoCloud --verify` (wrapped by
//  the zyquo-verify target and `make verify`). Keys come from environment
//  variables (source .env.keys); they are never logged or persisted here.
//
//  Per provider: /models diff vs catalog, a minimal "Reply with exactly: OK"
//  completion on EVERY catalog chat model, a streaming test, and a vision test
//  where supported. Results: stdout table + docs/VERIFICATION.md.
//

import Foundation

enum VerifyHarness {
    // MARK: - Result model

    struct TestResult {
        let provider: ProviderID
        let model: String
        let test: String
        let passed: Bool
        let latency: TimeInterval?
        let detail: String?
    }

    static let environmentKeys: [ProviderID: String] = [
        .openai: "OPENAI_API_KEY",
        .anthropic: "ANTHROPIC_API_KEY",
        .xai: "XAI_API_KEY",
        .mistral: "MISTRAL_API_KEY",
        .gemini: "GEMINI_API_KEY",
        .qwen: "DASHSCOPE_API_KEY",
        .deepseek: "DEEPSEEK_API_KEY",
        .kimi: "MOONSHOT_API_KEY",
        .perplexity: "PERPLEXITY_API_KEY",
        .together: "TOGETHER_API_KEY",
        .deepinfra: "DEEPINFRA_API_KEY",
        .cerebras: "CEREBRAS_API_KEY",
    ]

    /// Long-running or per-request-billed models excluded from the bulk "OK"
    /// sweep (each documented in docs/VERIFICATION.md).
    static let skipList: [String: String] = [
        "sonar-deep-research": "multi-minute agentic research runs; verified via docs only",
    ]

    /// 64×64 red PNG for vision tests (several providers reject tiny images).
    static let redPixelPNG = Data(base64Encoded:
        "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PQQkAAAgAsetfWiP4FgYrsKZeS0BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEDgsqnc8OJg6Ln3AAAAAElFTkSuQmCC"
    )!

    /// Model IDs that are documented rolling aliases — they resolve on the
    /// chat endpoint but never appear in the provider's /models listing.
    static let knownUnlistedAliases: Set<String> = [
        "grok-4.20", "grok-4.20-non-reasoning", "grok-code-fast-1",
    ]

    // MARK: - Entry

    static func run(arguments: [String]) async -> Int32 {
        let only = value(after: "--provider", in: arguments).flatMap { ProviderID(rawValue: $0) }
        let quick = arguments.contains("--quick")
        let providers = ProviderID.builtIn.filter { only == nil || $0 == only }

        let catalog = await MainActor.run { ModelCatalog() }
        let models = await MainActor.run { catalog.builtIn }

        print("Zyquo Cloud verification — \(providers.count) providers, \(models.count) catalog models\n")

        var missingKeys: [ProviderID] = []
        for provider in providers where apiKey(for: provider) == nil {
            missingKeys.append(provider)
        }
        if !missingKeys.isEmpty {
            print("⚠️  No key in environment for: \(missingKeys.map(\.rawValue).joined(separator: ", ")) — skipped\n")
        }

        let results = await withTaskGroup(of: [TestResult].self) { group in
            for provider in providers {
                guard let key = apiKey(for: provider) else { continue }
                let providerModels = models.filter { $0.provider == provider }
                group.addTask {
                    await verify(provider: provider, key: key, models: providerModels, quick: quick)
                }
            }
            var all: [TestResult] = []
            for await batch in group { all.append(contentsOf: batch) }
            return all
        }

        report(results: results, skipped: missingKeys)
        let failures = results.filter { !$0.passed }.count
        return failures == 0 ? 0 : 1
    }

    // MARK: - Per-provider verification

    private static func verify(
        provider: ProviderID, key: String, models: [AIModel], quick: Bool
    ) async -> [TestResult] {
        var results: [TestResult] = []
        let client = ProviderRegistry.client(for: provider)

        // 1. /models diff
        if provider.supportsModelListing {
            let start = Date()
            do {
                let live = Set(try await client.listModelIDs(apiKey: key))
                let missing = models.map(\.id).filter {
                    !live.contains($0) && !knownUnlistedAliases.contains($0)
                }
                results.append(TestResult(
                    provider: provider, model: "—", test: "models",
                    passed: missing.isEmpty,
                    latency: Date().timeIntervalSince(start),
                    detail: missing.isEmpty
                        ? "\(live.count) live"
                        : "catalog IDs not live: \(missing.joined(separator: ", "))"
                ))
            } catch {
                results.append(TestResult(
                    provider: provider, model: "—", test: "models",
                    passed: false, latency: nil, detail: error.localizedDescription
                ))
            }
        }

        // 2. "OK" sweep over every chat model (first model only in --quick mode)
        let sweep = quick ? Array(models.prefix(1)) : models
        // Serial for strict-rate-limit providers, else limited concurrency.
        let serial = provider == .cerebras || provider == .mistral
        if serial {
            for model in sweep {
                results.append(await okTest(client: client, model: model, key: key))
                if provider == .cerebras {
                    try? await Task.sleep(nanoseconds: 13_000_000_000) // free tier: 5 req/min
                } else {
                    try? await Task.sleep(nanoseconds: 1_100_000_000)
                }
            }
        } else {
            results.append(contentsOf: await limitedConcurrent(sweep, limit: 3) { model in
                await okTest(client: client, model: model, key: key)
            })
        }

        // 3. Streaming test on the first passing model
        if let streamModel = sweep.first(where: { model in
            results.contains { $0.model == model.id && $0.test == "chat" && $0.passed }
        }) ?? sweep.first {
            results.append(await streamTest(client: client, model: streamModel, key: key))
        }

        // 4. Vision test on the first vision-capable model
        if let visionModel = sweep.first(where: { $0.capabilities.vision && !$0.isLegacy }) {
            results.append(await visionTest(client: client, model: visionModel, key: key))
        }

        return results
    }

    // MARK: - Individual tests

    private static func okTest(client: ProviderClient, model: AIModel, key: String) async -> TestResult {
        if let reason = skipList[model.id] {
            return TestResult(
                provider: model.provider, model: model.id, test: "chat",
                passed: true, latency: nil, detail: "SKIPPED: \(reason)"
            )
        }
        let start = Date()
        var parameters = ChatParameters(maxTokens: model.capabilities.reasoning ? 4096 : 256)
        if model.parameterSupport.reasoningEffort {
            parameters.reasoningEffort = "low"
        }
        let request = ChatRequest(
            model: model,
            systemPrompt: nil,
            messages: [Message(role: .user, text: "Reply with exactly: OK")],
            parameters: parameters,
            stream: false
        )
        do {
            let reply = try await client.complete(request, apiKey: key)
            let text = reply.text.trimmingCharacters(in: .whitespacesAndNewlines)
            let passed = !text.isEmpty
            return TestResult(
                provider: model.provider, model: model.id, test: "chat",
                passed: passed,
                latency: Date().timeIntervalSince(start),
                detail: passed ? nil : "empty response"
            )
        } catch {
            return TestResult(
                provider: model.provider, model: model.id, test: "chat",
                passed: false, latency: Date().timeIntervalSince(start),
                detail: error.localizedDescription
            )
        }
    }

    private static func streamTest(client: ProviderClient, model: AIModel, key: String) async -> TestResult {
        let start = Date()
        var parameters = ChatParameters(maxTokens: model.capabilities.reasoning ? 4096 : 128)
        if model.parameterSupport.reasoningEffort { parameters.reasoningEffort = "low" }
        let request = ChatRequest(
            model: model,
            systemPrompt: nil,
            messages: [Message(role: .user, text: "Count from 1 to 5, digits separated by spaces.")],
            parameters: parameters
        )
        do {
            var deltas = 0
            var text = ""
            var sawUsage = false
            for try await event in client.streamChat(request, apiKey: key) {
                switch event {
                case .textDelta(let d): deltas += 1; text += d
                case .usage: sawUsage = true
                default: break
                }
            }
            let passed = deltas >= 1 && !text.isEmpty
            return TestResult(
                provider: model.provider, model: model.id, test: "stream",
                passed: passed,
                latency: Date().timeIntervalSince(start),
                detail: "deltas=\(deltas) usage=\(sawUsage)"
            )
        } catch {
            return TestResult(
                provider: model.provider, model: model.id, test: "stream",
                passed: false, latency: Date().timeIntervalSince(start),
                detail: error.localizedDescription
            )
        }
    }

    private static func visionTest(client: ProviderClient, model: AIModel, key: String) async -> TestResult {
        let start = Date()
        var message = Message(role: .user, text: "What color is this image? Reply with one word.")
        message.attachments = [
            Attachment(kind: .image, fileName: "pixel.png", data: redPixelPNG, mimeType: "image/png")
        ]
        var parameters = ChatParameters(maxTokens: model.capabilities.reasoning ? 2048 : 32)
        if model.parameterSupport.reasoningEffort { parameters.reasoningEffort = "low" }
        let request = ChatRequest(
            model: model, systemPrompt: nil, messages: [message],
            parameters: parameters, stream: false
        )
        do {
            let reply = try await client.complete(request, apiKey: key)
            let passed = !reply.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
            return TestResult(
                provider: model.provider, model: model.id, test: "vision",
                passed: passed,
                latency: Date().timeIntervalSince(start),
                detail: passed ? String(reply.text.prefix(40)) : "empty response"
            )
        } catch {
            return TestResult(
                provider: model.provider, model: model.id, test: "vision",
                passed: false, latency: Date().timeIntervalSince(start),
                detail: error.localizedDescription
            )
        }
    }

    // MARK: - Reporting

    private static func report(results: [TestResult], skipped: [ProviderID]) {
        var lines: [String] = []
        lines.append("<!--")
        lines.append("  VERIFICATION.md")
        lines.append("  Zyquo Cloud")
        lines.append("  Author: Simon-Pierre Boucher")
        lines.append("  Mail: contact@spboucher.ai")
        lines.append("-->")
        lines.append("")
        lines.append("# API Verification Results — \(ISO8601DateFormatter().string(from: Date()))")
        lines.append("")
        let failures = results.filter { !$0.passed }
        lines.append("**\(results.count) tests · \(results.count - failures.count) passed · \(failures.count) failed**")
        if !skipped.isEmpty {
            lines.append("")
            lines.append("Providers skipped (no key in environment): \(skipped.map(\.displayName).joined(separator: ", "))")
        }
        lines.append("")
        lines.append("| Provider | Model | Test | Result | Latency | Detail |")
        lines.append("|---|---|---|---|---|---|")
        let sorted = results.sorted {
            ($0.provider.rawValue, $0.test == "models" ? 0 : 1, $0.model, $0.test)
                < ($1.provider.rawValue, $1.test == "models" ? 0 : 1, $1.model, $1.test)
        }
        for result in sorted {
            let mark = result.passed ? "✅" : "❌"
            let latency = result.latency.map { String(format: "%.1fs", $0) } ?? "—"
            let detail = (result.detail ?? "").replacingOccurrences(of: "|", with: "\\|")
            lines.append("| \(result.provider.displayName) | `\(result.model)` | \(result.test) | \(mark) | \(latency) | \(detail) |")
        }
        lines.append("")
        let document = lines.joined(separator: "\n")

        let url = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
            .appendingPathComponent("docs/VERIFICATION.md")
        try? document.data(using: .utf8)?.write(to: url)

        // Console summary
        print(String(repeating: "—", count: 72))
        for result in sorted where !result.passed {
            print("❌ \(result.provider.rawValue) \(result.model) [\(result.test)] — \(result.detail ?? "")")
        }
        print(String(repeating: "—", count: 72))
        print("\(results.count) tests · \(results.count - failures.count) passed · \(failures.count) failed")
        print("Full table: docs/VERIFICATION.md")
    }

    // MARK: - Helpers

    static func apiKey(for provider: ProviderID) -> String? {
        guard let name = environmentKeys[provider] else { return nil }
        let value = ProcessInfo.processInfo.environment[name]
        return (value?.isEmpty ?? true) ? nil : value
    }

    /// Writes environment keys into the app's encrypted vault (for GUI use).
    static func loadVault() {
        let store = SecureKeyStore()
        var loaded: [String] = []
        for (provider, _) in environmentKeys {
            if let key = apiKey(for: provider) {
                try? store.setKey(key, for: provider)
                loaded.append(provider.rawValue)
            }
        }
        print("Vault updated with keys for: \(loaded.sorted().joined(separator: ", "))")
    }

    private static func value(after flag: String, in arguments: [String]) -> String? {
        guard let index = arguments.firstIndex(of: flag), index + 1 < arguments.count else { return nil }
        return arguments[index + 1]
    }

    private static func limitedConcurrent<T: Sendable>(
        _ models: [AIModel], limit: Int,
        _ operation: @escaping @Sendable (AIModel) async -> T
    ) async -> [T] {
        await withTaskGroup(of: (Int, T).self) { group in
            var results: [(Int, T)] = []
            var iterator = models.enumerated().makeIterator()
            var inFlight = 0
            func addNext(_ group: inout TaskGroup<(Int, T)>) {
                guard let (index, model) = iterator.next() else { return }
                inFlight += 1
                group.addTask { (index, await operation(model)) }
            }
            for _ in 0..<limit { addNext(&group) }
            while inFlight > 0 {
                guard let result = await group.next() else { break }
                inFlight -= 1
                results.append(result)
                addNext(&group)
            }
            return results.sorted { $0.0 < $1.0 }.map(\.1)
        }
    }
}