SPB Git

Repository Public

LiveTests.swift 23.4 KB 371 lines swift

1// Opt-in integration tests against a real backend (never production: the frozen test backend on M2U64:5058).2//3//   ssh -N -L 5058:localhost:5058 M2U64 &            # remember its PID, kill it afterwards4//   VQ_LIVE=1 VQ_BASE_URL=http://localhost:5058 VQ_TOKEN="$(ssh M2U64 cat /tmp/vq-api-token)" \5//     swift test --filter VQuantKitLiveTests6//7// Without VQ_LIVE=1 every test here is skipped. Objects created have ids prefixed `kit-live-` and are deleted at the end.8import Foundation9import Testing10import VQuantKit1112final class Counter: @unchecked Sendable {13    private let lock = NSLock()14    private var n = 015    func increment() { lock.withLock { n += 1 } }16    var value: Int { lock.withLock { n } }17}1819enum Live {20    static let env = ProcessInfo.processInfo.environment21    static var enabled: Bool { env["VQ_LIVE"] == "1" && token != nil }22    static var token: String? {23        if let t = env["VQ_TOKEN"], !t.isEmpty { return t }24        if let f = env["VQ_TOKEN_FILE"], let t = try? String(contentsOfFile: f, encoding: .utf8) { return t.trimmingCharacters(in: .whitespacesAndNewlines) }25        return nil26    }27    static func transport() -> VQURLSessionTransport {28        let token = self.token29        return VQURLSessionTransport(baseURL: URL(string: env["VQ_BASE_URL"] ?? "http://localhost:5058")!,30                                     clientHeader: VQuantAPI.macClientHeader(version: "2.0.0"), token: { token })31    }32    static func api() -> VQuantAPI { VQuantAPI(transport: transport()) }33    static func id(_ label: String) -> String { "kit-live-\(label)-\(UUID().uuidString.prefix(8).lowercased())" }3435    static func waitUntil(timeout: TimeInterval = 20, _ condition: @Sendable () async -> Bool) async -> Bool {36        let end = Date().addingTimeInterval(timeout)37        while Date() < end {38            if await condition() { return true }39            try? await Task.sleep(nanoseconds: 100_000_000)40        }41        return await condition()42    }43}4445@Suite(.enabled(if: Live.enabled), .serialized) struct VQuantKitLiveTests {46    @Test func objectRoundTripConflictVersionsAndTombstone() async throws {47        let store = Live.api().objects48        let id = Live.id("note")49        let created = try await store.put(.note, id, data: ["title": "Kit live", "body": "v1", "kitFutureField": ["keep": [1, 2, .null]]], baseRev: 0)50        #expect(created.rev == 1 && !created.deleted && created.seq > 0 && !created.device.isEmpty)51        let fetched = try await store.get(.note, id)52        #expect(fetched == created)53        #expect(fetched.data["kitFutureField"] == ["keep": [1, 2, .null]])                  // unknown keys are preserved by the real server5455        var note = try #require(fetched.typed(VQNote.self)).data56        note.body = "v2"57        let updated = try await store.put(.note, id, data: note.json, baseRev: 1)58        #expect(updated.rev == 2 && updated.seq > created.seq && updated.data["kitFutureField"] == fetched.data["kitFutureField"])5960        // a stale write is a 409 carrying the stored envelope61        do {62            _ = try await store.put(.note, id, data: ["title": "stale", "body": "x"], baseRev: 1)63            Issue.record("a stale write was accepted")64        } catch let e as VQError {65            #expect(e.kind == .conflict && VQObjectStoreClient.current(of: e) == updated)66        }67        // creating over a live object is a conflict too68        do { _ = try await store.put(.note, id, data: ["title": "t", "body": "b"], baseRev: 0); Issue.record("created twice") }69        catch let e as VQError { #expect(e.kind == .conflict) }7071        let versions = try await store.versions(.note, id)72        #expect(versions.map(\.rev) == [1])73        #expect(try await store.version(.note, id, rev: 1).data["body"] == "v1")74        #expect(try await store.list(.note).contains { $0.id == id })7576        // validation, server-only kind, unknown kind77        do { _ = try await store.put(.note, Live.id("bad"), data: ["title": 5], baseRev: 0); Issue.record("invalid data accepted") }78        catch let e as VQError { #expect(e.kind == .rejected && !e.message.isEmpty) }79        do { _ = try await store.put(.review, Live.id("rev"), data: [:], baseRev: 0); Issue.record("review written by a client") }80        catch let e as VQError { #expect(e.kind == .forbidden) }81        do { _ = try await store.list("no_such_kind"); Issue.record("unknown kind listed") }82        catch let e as VQError { #expect(e.kind == .notFound) }8384        let tomb = try await store.delete(.note, id, baseRev: 2)85        #expect(tomb.deleted && tomb.rev == 3 && tomb.data == .object([:]))86        do { _ = try await store.get(.note, id); Issue.record("a deleted object was served") }87        catch let e as VQError { #expect(e.kind == .notFound) }88        // any baseRev other than 0 on a tombstone is a conflict carrying the tombstone89        do { _ = try await store.put(.note, id, data: note.json, baseRev: 3); Issue.record("wrote over a tombstone with its rev") }90        catch let e as VQError { #expect(e.kind == .conflict && VQObjectStoreClient.current(of: e)?.deleted == true) }91        let again = try await store.put(.note, id, data: note.json, baseRev: 0)92        #expect(again.rev == 4 && again.createdAt >= created.createdAt)                       // the revision count continues93        _ = try await store.delete(.note, id, baseRev: 4)94    }9596    @Test func changesFeedPagesFromACursor() async throws {97        let store = Live.api().objects98        let start = try await store.changes(since: 0, limit: 1)99        let cursor = start.seq100        let ids = (0..<3).map { _ in Live.id("wl") }101        for (i, id) in ids.enumerated() { _ = try await store.put(.watchlist, id, data: ["name": .string("Kit live \(i)"), "symbols": ["AAPL"]], baseRev: 0) }102        _ = try await store.put(.watchlist, ids[0], data: ["name": "Kit live 0b", "symbols": ["AAPL", "MSFT"]], baseRev: 1)103        // latest state, one entry per object, ascending seq104        let first = try await store.changes(since: cursor, kinds: [.watchlist], limit: 2)105        #expect(first.changes.count == 2 && first.more)106        let second = try await store.changes(since: first.changes.last!.seq, kinds: [.watchlist], limit: 2)107        let mine = (first.changes + second.changes).filter { ids.contains($0.id) }108        #expect(mine.map(\.id) == [ids[1], ids[2], ids[0]])109        #expect(mine.last?.rev == 2 && !second.more && second.seq >= mine.last!.seq)110        do { _ = try await store.changes(since: 0, kinds: ["nope"]); Issue.record("unknown kind filter accepted") }111        catch let e as VQError { #expect(e.kind == .rejected) }112        for (i, id) in ids.enumerated() { _ = try await store.delete(.watchlist, id, baseRev: i == 0 ? 2 : 1) }113        let after = try await store.changes(since: second.seq, kinds: [.watchlist])114        #expect(after.changes.filter { ids.contains($0.id) }.allSatisfy { $0.deleted })          // tombstones are in the feed115    }116117    /// Two clients through the real server: a stream receives a change made by the other one, and a concurrent edit118    /// is merged as the contract prescribes.119    @Test func twoEnginesStreamAndMerge() async throws {120        let a = VQSyncEngine(transport: Live.transport(), storage: VQMemorySyncStorage())121        var manual = VQSyncEngine.Configuration(); manual.autoFlush = false122        let b = VQSyncEngine(transport: Live.transport(), storage: VQMemorySyncStorage(), configuration: manual)123        let id = Live.id("wl")124        let hints = await a.hints()125        await a.start()126        #expect(await Live.waitUntil { await a.status.connection == .connected })127        #expect(await Live.waitUntil { await a.status.lastPulledAt != nil })128129        try await b.pull()130        try await b.put(VQWatchlist(name: "Kit live stream", symbols: ["AAPL", "MSFT"]), id: id)131        await b.flush()132        #expect(await b.status.pendingWrites == 0)133        #expect(await Live.waitUntil { await a.object(VQWatchlist.self, id: id)?.data.symbols == ["AAPL", "MSFT"] }, "the change did not arrive on the stream")134135        // concurrent edits: B writes first, A (which will not have pulled B's write yet in its queue) merges136        await a.stop()137        var wb = try #require(await b.object(VQWatchlist.self, id: id)).data138        wb.add("NVDA"); wb.remove("MSFT")139        try await b.put(wb, id: id); await b.flush()140        var wa = try #require(await a.object(VQWatchlist.self, id: id)).data141        wa.add("AMZN")142        try await a.put(wa, id: id)143        await a.flush()144        #expect(await Live.waitUntil { await a.status.pendingWrites == 0 })145        let merged = try await Live.api().objects.get(.watchlist, id)146        #expect(merged.data["symbols"] == ["AAPL", "AMZN", "NVDA"] && merged.rev == 3)        // union, each side's removal holds147        #expect(await a.conflicts.contains { $0.key.id == id && $0.resolution == .mergedElements })148        try await b.pull()149        #expect(await b.object(.watchlist, id)?.data == merged.data)150151        // a quant job started by "another device" produces a hint on the stream152        await a.start()153        #expect(await Live.waitUntil { await a.status.connection == .connected })154        let lab = Live.api().lab155        let job = try await lab.start(tool: "options_lab", params: ["symbol": "SPY", "mode": "price"])156        let got = await withTaskGroup(of: VQSyncHint?.self) { group -> VQSyncHint? in157            group.addTask { for await h in hints where h.topic == .quantJobs && h.id == job.id { return h }; return nil }158            group.addTask { try? await Task.sleep(nanoseconds: 20_000_000_000); return nil }159            let first = await group.next() ?? nil160            group.cancelAll()161            return first162        }163        #expect(got?.id == job.id, "no quant_jobs hint for the job")164165        try await a.delete(.watchlist, id)166        #expect(await Live.waitUntil { await a.status.pendingWrites == 0 })167        await a.stop()168        _ = try? await lab.delete(job.id)169    }170171    @Test func labJobToCompletionAndCancel() async throws {172        let lab = Live.api().lab173        let catalog = try await lab.catalog()174        #expect(catalog.tools.count == 19 && catalog.labEngines.count == 13)175176        var form = VQLabForm(tool: try #require(catalog.tool("bootstrap")))177        form["symbol"] = "AAPL"; form["replications"] = 500178        guard case .success(let params) = form.validate() else { Issue.record("form invalid"); return }179        let states = Counter()180        let done = try await lab.run(tool: "bootstrap", params: params, every: 0.5) { _ in states.increment() }181        #expect(done.status == .done && done.progress == 1 && states.value >= 2)182        #expect(done.params == params)                                                       // the server recorded exactly what the form built183        let result = try #require(done.labResult)184        #expect(result.success && result.engine == "bootstrap" && !result.summary.isEmpty && !result.charts.isEmpty && result.provenance?.seed == 12345)185        #expect(try await lab.jobs().jobs.contains { $0.id == done.id })186187        // the server validates too: its sentence comes back188        do { _ = try await lab.start(tool: "bootstrap", params: ["symbol": "AAPL", "replications": 5]); Issue.record("invalid params accepted") }189        catch let e as VQError { #expect(e.kind == .rejected && e.message.contains("must be between")) }190191        // cancel a long one192        let long = try await lab.start(tool: "bootstrap", params: ["symbol": "MSFT", "replications": 20000])193        #expect(long.isRunning && long.cancellable)194        try await Task.sleep(nanoseconds: 700_000_000)195        guard case .cancelled(let cancelled) = try await lab.cancel(long.id) else { Issue.record("cancel refused"); return }196        #expect(cancelled.status == .cancelled)197        let after = try await lab.job(long.id)198        #expect(after.status == .cancelled && after.result == nil)199        #expect(try await lab.cancel(long.id) == .notRunning(status: "cancelled"))200        // a legacy tool cannot be cancelled201        let legacy = try await lab.start(tool: "garch", params: ["symbol": "AAPL"])202        if legacy.isRunning { #expect(try await lab.cancel(legacy.id) == .notCancellable) }203        var last = legacy204        for try await s in lab.follow(legacy.id, every: 0.5) { last = s }205        #expect(last.status == .done && last.labResult == nil && last.result != nil)206        for id in [done.id, long.id, legacy.id] { try? await lab.delete(id) }207    }208209    @Test func modelsRoutingAndUsage() async throws {210        let models = Live.api().models211        let catalog = try await models.catalog()212        #expect(catalog.providers.count == 5 && catalog.model(catalog.defaultModel)?.selectable == true)213        let route = try await models.route(mode: "fast", vision: false, query: "Cours de AAPL ?")214        #expect(catalog.model(route.model)?.selectable == true && !route.reason.isEmpty && route.candidates.first?.model == route.model)215        let usage = try await models.usage(days: 30)216        #expect(usage.days == 30 && usage.totals.inputTokens >= 0)217    }218219    @Test func marketDataSurface() async throws {220        let data = Live.api().data221        let overview = try await data.marketOverview()222        #expect(overview.sections.count >= 8 && overview.section("indices")?.items.isEmpty == false && overview.yieldCurve?.points.isEmpty == false)223        let quotes = try await data.quotes(["AAPL", "^GSPC", "EURUSD", "ZZZZNOPE"])224        #expect(quotes["AAPL"]?.price != nil && quotes["^GSPC"] != nil && quotes.missing == ["ZZZZNOPE"] && quotes.freshness?.kind == .polled)225        let bars = try await data.bars("AAPL", interval: "1d", limit: 30)226        #expect(bars.bars.count == 30 && bars.provenance.provider == "fmp" && bars.provenance.freshness != nil)227        do { _ = try await data.bars("ZZZZNOPE"); Issue.record("bars for a symbol that does not exist") }228        catch let e as VQError { #expect(e.kind == .notFound && !VQDataClient.providerReasons(of: e).isEmpty) }229        let statements = try await data.statements("AAPL", period: "quarter", limit: 2)230        #expect(statements.periods.count == 2 && statements.income.first?.values.count == 2)231        #expect(try await data.health().providers.first?.id == "fmp")232        let search = try await data.search("AAPL", types: [.symbol, .action], limit: 5)233        #expect(search.results.first?.symbol == "AAPL" && search.results.count <= 5)234    }235236    @Test func researchSurface() async throws {237        let api = Live.api()238        let pid = Live.id("project"), nid = Live.id("note")239        _ = try await api.objects.put(.note, nid, data: VQNote(title: "Kit live note", body: "Body").json, baseRev: 0)240        _ = try await api.objects.put(.project, pid, data: VQProject(title: "Kit live project", items: [.init(id: "i1", type: .note, ref: nid), .init(id: "i2", type: .dataset, ref: "none")]).json, baseRev: 0)241        let bundle = try await api.research.bundle(project: pid)242        #expect(bundle.data.title == "Kit live project" && bundle.items.count == 2 && bundle.items[0].resolved?.title == "Kit live note" && bundle.items[1].resolved == nil)243        let export = try await api.research.export(project: pid, format: .md)244        #expect(export.format == "md" && export.url.hasPrefix("/api/download/") && (export.bytes ?? 0) > 0)245        let audit = try await api.research.audit(limit: 5)246        #expect(audit.contains { $0.action == "project.export" && $0.target == "project/\(pid)" })247        let memory = try await api.research.memory()248        #expect(memory.groups.first?.scope == .user)249        _ = try await api.objects.delete(.project, pid, baseRev: 1)250        _ = try await api.objects.delete(.note, nid, baseRev: 1)251        #expect(try await api.research.audit(limit: 5).contains { $0.action == "object.delete" && $0.target == "note/\(nid)" })252    }253254    /// 0.3: the form's conditional rule gives the very sentence the server answers, and new results carry the renderer fields.255    @Test func conditionalParametersAndRendererFields() async throws {256        let api = Live.api()257        let options = try #require(try await api.lab.catalog().tool("options_lab"))258        var form = VQLabForm(tool: options)259        form["mode"] = "implied"260        guard case .failure(let invalid) = form.validate() else { Issue.record("the form accepted an implied-volatility request without a price"); return }261        do { _ = try await api.lab.start(tool: "options_lab", params: ["symbol": "SPY", "mode": "implied"]); Issue.record("the server accepted it") }262        catch let e as VQError { #expect(e.kind == .rejected && e.message == invalid.issues.first?.message, "server: \(e.message) — form: \(invalid.message)") }263        form["mode"] = "price"264        guard case .success(let params) = form.validate() else { Issue.record("defaults invalid"); return }265        let job = try await api.lab.run(tool: "options_lab", params: params, every: 0.5)266        let result = try #require(job.labResult)267        #expect(result.summary.filter { $0.unit == .currency }.allSatisfy { $0.currency != nil } && result.tables.flatMap(\.columns).allSatisfy { $0.type != nil })268        #expect(result.charts.allSatisfy { $0.y.unit != .currency || $0.y.currency != nil })269        try? await api.lab.delete(job.id)270        #expect(await api.researchAgent.researchModeAvailability() != nil)                // the sandbox route answers: offer the research mode271        #expect(try await api.models.usage(days: 30).byModel.allSatisfy { $0.role != nil })272    }273274    @Test func sandboxStatusIsServed() async throws {275        let sandbox = try await Live.api().researchAgent.sandbox()276        #expect(sandbox.uploadMaxBytes > 0 && !sandbox.limits.isEmpty && !sandbox.defaultNetwork.isEmpty)277        #expect(sandbox.enforced || sandbox.note != nil)278        _ = try await Live.api().researchAgent.approvals()279    }280}281282/// The research agent's REST against a backend that holds a real study (the 5059 copy):283///284///   ssh -f -N -L 5059:localhost:5059 M2U64285///   VQ_LIVE=1 VQ_RESEARCH_BASE_URL=http://localhost:5059 VQ_RESEARCH_TOKEN="$(ssh M2U64 cat /tmp/vq-api-token-5059)" swift test --filter VQuantKitResearchLiveTests286///287/// Read-only: it lists sessions, reads a workspace tree, a file, the ledger, runs and logs. It never starts a study.288@Suite(.enabled(if: Live.env["VQ_LIVE"] == "1" && (Live.env["VQ_RESEARCH_TOKEN"] ?? "").hasPrefix("vqn_")), .serialized) struct VQuantKitResearchLiveTests {289    private func client() -> VQResearchAgentClient {290        let token = Live.env["VQ_RESEARCH_TOKEN"]291        let t = VQURLSessionTransport(baseURL: URL(string: Live.env["VQ_RESEARCH_BASE_URL"] ?? "http://localhost:5059")!, clientHeader: VQuantAPI.macClientHeader(version: "2.0.0"), token: { token })292        return VQuantAPI(transport: t).researchAgent293    }294295    @Test func sessionsTreeFileAndLedger() async throws {296        let c = client()297        let sessions = try await c.sessions()298        // the study with the most work in it299        var best: (VQResearchSession, VQResearchSessionDetail)?300        for s in sessions.prefix(8) {301            let d = try await c.session(s.id)302            if d.executions.count > (best?.1.executions.count ?? -1) { best = (s, d) }303        }304        let (session, detail) = try #require(best, "no research session on this backend")305        #expect(!detail.files.isEmpty && detail.session.id == session.id && detail.environment.hash != nil)306307        // the same workspace by conversation308        if let sid = session.sessionIds.first { #expect(try await c.session(VQResearchSession.reference(sessionId: sid)).session.id == session.id) }309310        let files = try await c.files(session.id)311        #expect(files.count == detail.files.count && !VQResearchFileTree.build(files).isEmpty)312        let code = try #require(files.first { !$0.isDirectory && $0.path.hasSuffix(".py") })313        let preview = try await c.textFile(session.id, path: code.path, offset: 0, limit: 5)314        #expect(preview.path == code.path && preview.to <= 5 && !preview.text.isEmpty && preview.totalLines >= preview.to)315        let bytes = try await c.fileData(session.id, path: code.path)316        #expect(bytes.count == code.bytes)317        if let png = files.first(where: { $0.path.hasSuffix(".png") }) {318            let data = try await c.fileData(session.id, path: png.path)319            #expect(data.prefix(4) == Data([0x89, 0x50, 0x4E, 0x47]))320        }321        do { _ = try await c.textFile(session.id, path: "no/such/file.py"); Issue.record("a missing file was served") }322        catch let e as VQError { #expect(e.kind == .notFound) }323324        let experiments = try await c.experiments(session.id, kind: .experiment)325        let executions = try await c.experiments(session.id, kind: .execution)326        #expect(experiments.count == detail.experiments.count && executions.count == detail.executions.count)327        #expect(experiments.allSatisfy { $0.kind == .experiment && $0.status.known != nil } && executions.allSatisfy { $0.kind == .execution })328        #expect(zip(executions, executions.dropFirst()).allSatisfy { ($0.seq ?? 0) < ($1.seq ?? 0) })329        let runs = try await c.runs(session.id)330        #expect(runs.count == detail.runs.count)331        if let run = runs.first {332            let logs = try await c.run(run.id, tail: 2000)333            #expect(logs.run.id == run.id && logs.stdout.count <= 2100)334        }335        if let data = files.first(where: { $0.path.hasPrefix("data/") && !$0.isDirectory }) {336            let inspection = try await c.inspect(session.id, path: data.path)337            #expect(inspection.path == data.path && !inspection.columns.isEmpty && (inspection.rows ?? 0) > 0)338        }339        // 0.3: generations with their plan, and — on studies recent enough — the figures with their verdict340        #expect(!detail.generations.isEmpty && detail.generations.allSatisfy { $0.status.known != nil && !$0.generationId.isEmpty })341        var sawFigureRecord = false, sawPlan = false342        for s in sessions.prefix(8) {343            let d = try await c.session(s.id)344            if !d.plan.isEmpty { sawPlan = true; #expect(VQResearchState(detail: d).plan == d.plan) }345            for f in d.figures {346                sawFigureRecord = true347                #expect(f.status?.known != nil && f.files?.png != nil)348                let rebuilt = try #require(VQResearchState(detail: d).figures.first { $0.figureId == f.id })349                #expect(rebuilt.lintStatus == f.status && rebuilt.caption == f.caption)350                if let png = f.files?.png { #expect(try await c.fileData(s.id, path: png).prefix(4) == Data([0x89, 0x50, 0x4E, 0x47])) }351            }352        }353        #expect(sawPlan && sawFigureRecord)354        // answered approvals are listed with their workspace355        let approvals = try await c.approvals(pendingOnly: false)356        #expect(approvals.allSatisfy { $0.workspaceId != nil && $0.status.known != nil })357        if let answered = approvals.first(where: { $0.status == .approved }), let ws = answered.workspaceId {358            let d = try await c.session(ws)359            #expect(VQResearchState(detail: d, approvals: approvals).approvals.contains { $0.id == answered.id })360            do { _ = try await c.answer(approval: answered.id, .deny); Issue.record("an answered approval was answered again") }361            catch let e as VQError { #expect(e.kind == .notFound) }362        }363        // the study rebuilt from REST alone364        let state = VQResearchState(detail: detail)365        #expect(state.counts.executions == detail.executions.count && state.experiments.count == detail.experiments.count && state.finished)366        #expect(try await c.sandbox().uploadMaxBytes > 0)367        do { _ = try await c.session("s-does-not-exist"); Issue.record("an unknown workspace was served") }368        catch let e as VQError { #expect(e.kind == .notFound) }369    }370}371
Files main
Menu

Theme