Repository Public
1// Opt-in integration tests against a real backend (never production: the frozen test backend on M2U64:5058).2//3// ssh -N -L 5058:localhost:5058 M2U64 & # remember its PID, kill it afterwards4// VQ_LIVE=1 VQ_BASE_URL=http://localhost:5058 VQ_TOKEN="$(ssh M2U64 cat /tmp/vq-api-token)" \5// swift test --filter VQuantKitLiveTests6//7// Without VQ_LIVE=1 every test here is skipped. Objects created have ids prefixed `kit-live-` and are deleted at the end.8import Foundation9import Testing10import VQuantKit1112final class Counter: @unchecked Sendable {13 private let lock = NSLock()14 private var n = 015 func increment() { lock.withLock { n += 1 } }16 var value: Int { lock.withLock { n } }17}1819enum Live {20 static let env = ProcessInfo.processInfo.environment21 static var enabled: Bool { env["VQ_LIVE"] == "1" && token != nil }22 static var token: String? {23 if let t = env["VQ_TOKEN"], !t.isEmpty { return t }24 if let f = env["VQ_TOKEN_FILE"], let t = try? String(contentsOfFile: f, encoding: .utf8) { return t.trimmingCharacters(in: .whitespacesAndNewlines) }25 return nil26 }27 static func transport() -> VQURLSessionTransport {28 let token = self.token29 return VQURLSessionTransport(baseURL: URL(string: env["VQ_BASE_URL"] ?? "http://localhost:5058")!,30 clientHeader: VQuantAPI.macClientHeader(version: "2.0.0"), token: { token })31 }32 static func api() -> VQuantAPI { VQuantAPI(transport: transport()) }33 static func id(_ label: String) -> String { "kit-live-\(label)-\(UUID().uuidString.prefix(8).lowercased())" }3435 static func waitUntil(timeout: TimeInterval = 20, _ condition: @Sendable () async -> Bool) async -> Bool {36 let end = Date().addingTimeInterval(timeout)37 while Date() < end {38 if await condition() { return true }39 try? await Task.sleep(nanoseconds: 100_000_000)40 }41 return await condition()42 }43}4445@Suite(.enabled(if: Live.enabled), .serialized) struct VQuantKitLiveTests {46 @Test func objectRoundTripConflictVersionsAndTombstone() async throws {47 let store = Live.api().objects48 let id = Live.id("note")49 let created = try await store.put(.note, id, data: ["title": "Kit live", "body": "v1", "kitFutureField": ["keep": [1, 2, .null]]], baseRev: 0)50 #expect(created.rev == 1 && !created.deleted && created.seq > 0 && !created.device.isEmpty)51 let fetched = try await store.get(.note, id)52 #expect(fetched == created)53 #expect(fetched.data["kitFutureField"] == ["keep": [1, 2, .null]]) // unknown keys are preserved by the real server5455 var note = try #require(fetched.typed(VQNote.self)).data56 note.body = "v2"57 let updated = try await store.put(.note, id, data: note.json, baseRev: 1)58 #expect(updated.rev == 2 && updated.seq > created.seq && updated.data["kitFutureField"] == fetched.data["kitFutureField"])5960 // a stale write is a 409 carrying the stored envelope61 do {62 _ = try await store.put(.note, id, data: ["title": "stale", "body": "x"], baseRev: 1)63 Issue.record("a stale write was accepted")64 } catch let e as VQError {65 #expect(e.kind == .conflict && VQObjectStoreClient.current(of: e) == updated)66 }67 // creating over a live object is a conflict too68 do { _ = try await store.put(.note, id, data: ["title": "t", "body": "b"], baseRev: 0); Issue.record("created twice") }69 catch let e as VQError { #expect(e.kind == .conflict) }7071 let versions = try await store.versions(.note, id)72 #expect(versions.map(\.rev) == [1])73 #expect(try await store.version(.note, id, rev: 1).data["body"] == "v1")74 #expect(try await store.list(.note).contains { $0.id == id })7576 // validation, server-only kind, unknown kind77 do { _ = try await store.put(.note, Live.id("bad"), data: ["title": 5], baseRev: 0); Issue.record("invalid data accepted") }78 catch let e as VQError { #expect(e.kind == .rejected && !e.message.isEmpty) }79 do { _ = try await store.put(.review, Live.id("rev"), data: [:], baseRev: 0); Issue.record("review written by a client") }80 catch let e as VQError { #expect(e.kind == .forbidden) }81 do { _ = try await store.list("no_such_kind"); Issue.record("unknown kind listed") }82 catch let e as VQError { #expect(e.kind == .notFound) }8384 let tomb = try await store.delete(.note, id, baseRev: 2)85 #expect(tomb.deleted && tomb.rev == 3 && tomb.data == .object([:]))86 do { _ = try await store.get(.note, id); Issue.record("a deleted object was served") }87 catch let e as VQError { #expect(e.kind == .notFound) }88 // any baseRev other than 0 on a tombstone is a conflict carrying the tombstone89 do { _ = try await store.put(.note, id, data: note.json, baseRev: 3); Issue.record("wrote over a tombstone with its rev") }90 catch let e as VQError { #expect(e.kind == .conflict && VQObjectStoreClient.current(of: e)?.deleted == true) }91 let again = try await store.put(.note, id, data: note.json, baseRev: 0)92 #expect(again.rev == 4 && again.createdAt >= created.createdAt) // the revision count continues93 _ = try await store.delete(.note, id, baseRev: 4)94 }9596 @Test func changesFeedPagesFromACursor() async throws {97 let store = Live.api().objects98 let start = try await store.changes(since: 0, limit: 1)99 let cursor = start.seq100 let ids = (0..<3).map { _ in Live.id("wl") }101 for (i, id) in ids.enumerated() { _ = try await store.put(.watchlist, id, data: ["name": .string("Kit live \(i)"), "symbols": ["AAPL"]], baseRev: 0) }102 _ = try await store.put(.watchlist, ids[0], data: ["name": "Kit live 0b", "symbols": ["AAPL", "MSFT"]], baseRev: 1)103 // latest state, one entry per object, ascending seq104 let first = try await store.changes(since: cursor, kinds: [.watchlist], limit: 2)105 #expect(first.changes.count == 2 && first.more)106 let second = try await store.changes(since: first.changes.last!.seq, kinds: [.watchlist], limit: 2)107 let mine = (first.changes + second.changes).filter { ids.contains($0.id) }108 #expect(mine.map(\.id) == [ids[1], ids[2], ids[0]])109 #expect(mine.last?.rev == 2 && !second.more && second.seq >= mine.last!.seq)110 do { _ = try await store.changes(since: 0, kinds: ["nope"]); Issue.record("unknown kind filter accepted") }111 catch let e as VQError { #expect(e.kind == .rejected) }112 for (i, id) in ids.enumerated() { _ = try await store.delete(.watchlist, id, baseRev: i == 0 ? 2 : 1) }113 let after = try await store.changes(since: second.seq, kinds: [.watchlist])114 #expect(after.changes.filter { ids.contains($0.id) }.allSatisfy { $0.deleted }) // tombstones are in the feed115 }116117 /// Two clients through the real server: a stream receives a change made by the other one, and a concurrent edit118 /// is merged as the contract prescribes.119 @Test func twoEnginesStreamAndMerge() async throws {120 let a = VQSyncEngine(transport: Live.transport(), storage: VQMemorySyncStorage())121 var manual = VQSyncEngine.Configuration(); manual.autoFlush = false122 let b = VQSyncEngine(transport: Live.transport(), storage: VQMemorySyncStorage(), configuration: manual)123 let id = Live.id("wl")124 let hints = await a.hints()125 await a.start()126 #expect(await Live.waitUntil { await a.status.connection == .connected })127 #expect(await Live.waitUntil { await a.status.lastPulledAt != nil })128129 try await b.pull()130 try await b.put(VQWatchlist(name: "Kit live stream", symbols: ["AAPL", "MSFT"]), id: id)131 await b.flush()132 #expect(await b.status.pendingWrites == 0)133 #expect(await Live.waitUntil { await a.object(VQWatchlist.self, id: id)?.data.symbols == ["AAPL", "MSFT"] }, "the change did not arrive on the stream")134135 // concurrent edits: B writes first, A (which will not have pulled B's write yet in its queue) merges136 await a.stop()137 var wb = try #require(await b.object(VQWatchlist.self, id: id)).data138 wb.add("NVDA"); wb.remove("MSFT")139 try await b.put(wb, id: id); await b.flush()140 var wa = try #require(await a.object(VQWatchlist.self, id: id)).data141 wa.add("AMZN")142 try await a.put(wa, id: id)143 await a.flush()144 #expect(await Live.waitUntil { await a.status.pendingWrites == 0 })145 let merged = try await Live.api().objects.get(.watchlist, id)146 #expect(merged.data["symbols"] == ["AAPL", "AMZN", "NVDA"] && merged.rev == 3) // union, each side's removal holds147 #expect(await a.conflicts.contains { $0.key.id == id && $0.resolution == .mergedElements })148 try await b.pull()149 #expect(await b.object(.watchlist, id)?.data == merged.data)150151 // a quant job started by "another device" produces a hint on the stream152 await a.start()153 #expect(await Live.waitUntil { await a.status.connection == .connected })154 let lab = Live.api().lab155 let job = try await lab.start(tool: "options_lab", params: ["symbol": "SPY", "mode": "price"])156 let got = await withTaskGroup(of: VQSyncHint?.self) { group -> VQSyncHint? in157 group.addTask { for await h in hints where h.topic == .quantJobs && h.id == job.id { return h }; return nil }158 group.addTask { try? await Task.sleep(nanoseconds: 20_000_000_000); return nil }159 let first = await group.next() ?? nil160 group.cancelAll()161 return first162 }163 #expect(got?.id == job.id, "no quant_jobs hint for the job")164165 try await a.delete(.watchlist, id)166 #expect(await Live.waitUntil { await a.status.pendingWrites == 0 })167 await a.stop()168 _ = try? await lab.delete(job.id)169 }170171 @Test func labJobToCompletionAndCancel() async throws {172 let lab = Live.api().lab173 let catalog = try await lab.catalog()174 #expect(catalog.tools.count == 19 && catalog.labEngines.count == 13)175176 var form = VQLabForm(tool: try #require(catalog.tool("bootstrap")))177 form["symbol"] = "AAPL"; form["replications"] = 500178 guard case .success(let params) = form.validate() else { Issue.record("form invalid"); return }179 let states = Counter()180 let done = try await lab.run(tool: "bootstrap", params: params, every: 0.5) { _ in states.increment() }181 #expect(done.status == .done && done.progress == 1 && states.value >= 2)182 #expect(done.params == params) // the server recorded exactly what the form built183 let result = try #require(done.labResult)184 #expect(result.success && result.engine == "bootstrap" && !result.summary.isEmpty && !result.charts.isEmpty && result.provenance?.seed == 12345)185 #expect(try await lab.jobs().jobs.contains { $0.id == done.id })186187 // the server validates too: its sentence comes back188 do { _ = try await lab.start(tool: "bootstrap", params: ["symbol": "AAPL", "replications": 5]); Issue.record("invalid params accepted") }189 catch let e as VQError { #expect(e.kind == .rejected && e.message.contains("must be between")) }190191 // cancel a long one192 let long = try await lab.start(tool: "bootstrap", params: ["symbol": "MSFT", "replications": 20000])193 #expect(long.isRunning && long.cancellable)194 try await Task.sleep(nanoseconds: 700_000_000)195 guard case .cancelled(let cancelled) = try await lab.cancel(long.id) else { Issue.record("cancel refused"); return }196 #expect(cancelled.status == .cancelled)197 let after = try await lab.job(long.id)198 #expect(after.status == .cancelled && after.result == nil)199 #expect(try await lab.cancel(long.id) == .notRunning(status: "cancelled"))200 // a legacy tool cannot be cancelled201 let legacy = try await lab.start(tool: "garch", params: ["symbol": "AAPL"])202 if legacy.isRunning { #expect(try await lab.cancel(legacy.id) == .notCancellable) }203 var last = legacy204 for try await s in lab.follow(legacy.id, every: 0.5) { last = s }205 #expect(last.status == .done && last.labResult == nil && last.result != nil)206 for id in [done.id, long.id, legacy.id] { try? await lab.delete(id) }207 }208209 @Test func modelsRoutingAndUsage() async throws {210 let models = Live.api().models211 let catalog = try await models.catalog()212 #expect(catalog.providers.count == 5 && catalog.model(catalog.defaultModel)?.selectable == true)213 let route = try await models.route(mode: "fast", vision: false, query: "Cours de AAPL ?")214 #expect(catalog.model(route.model)?.selectable == true && !route.reason.isEmpty && route.candidates.first?.model == route.model)215 let usage = try await models.usage(days: 30)216 #expect(usage.days == 30 && usage.totals.inputTokens >= 0)217 }218219 @Test func marketDataSurface() async throws {220 let data = Live.api().data221 let overview = try await data.marketOverview()222 #expect(overview.sections.count >= 8 && overview.section("indices")?.items.isEmpty == false && overview.yieldCurve?.points.isEmpty == false)223 let quotes = try await data.quotes(["AAPL", "^GSPC", "EURUSD", "ZZZZNOPE"])224 #expect(quotes["AAPL"]?.price != nil && quotes["^GSPC"] != nil && quotes.missing == ["ZZZZNOPE"] && quotes.freshness?.kind == .polled)225 let bars = try await data.bars("AAPL", interval: "1d", limit: 30)226 #expect(bars.bars.count == 30 && bars.provenance.provider == "fmp" && bars.provenance.freshness != nil)227 do { _ = try await data.bars("ZZZZNOPE"); Issue.record("bars for a symbol that does not exist") }228 catch let e as VQError { #expect(e.kind == .notFound && !VQDataClient.providerReasons(of: e).isEmpty) }229 let statements = try await data.statements("AAPL", period: "quarter", limit: 2)230 #expect(statements.periods.count == 2 && statements.income.first?.values.count == 2)231 #expect(try await data.health().providers.first?.id == "fmp")232 let search = try await data.search("AAPL", types: [.symbol, .action], limit: 5)233 #expect(search.results.first?.symbol == "AAPL" && search.results.count <= 5)234 }235236 @Test func researchSurface() async throws {237 let api = Live.api()238 let pid = Live.id("project"), nid = Live.id("note")239 _ = try await api.objects.put(.note, nid, data: VQNote(title: "Kit live note", body: "Body").json, baseRev: 0)240 _ = try await api.objects.put(.project, pid, data: VQProject(title: "Kit live project", items: [.init(id: "i1", type: .note, ref: nid), .init(id: "i2", type: .dataset, ref: "none")]).json, baseRev: 0)241 let bundle = try await api.research.bundle(project: pid)242 #expect(bundle.data.title == "Kit live project" && bundle.items.count == 2 && bundle.items[0].resolved?.title == "Kit live note" && bundle.items[1].resolved == nil)243 let export = try await api.research.export(project: pid, format: .md)244 #expect(export.format == "md" && export.url.hasPrefix("/api/download/") && (export.bytes ?? 0) > 0)245 let audit = try await api.research.audit(limit: 5)246 #expect(audit.contains { $0.action == "project.export" && $0.target == "project/\(pid)" })247 let memory = try await api.research.memory()248 #expect(memory.groups.first?.scope == .user)249 _ = try await api.objects.delete(.project, pid, baseRev: 1)250 _ = try await api.objects.delete(.note, nid, baseRev: 1)251 #expect(try await api.research.audit(limit: 5).contains { $0.action == "object.delete" && $0.target == "note/\(nid)" })252 }253254 /// 0.3: the form's conditional rule gives the very sentence the server answers, and new results carry the renderer fields.255 @Test func conditionalParametersAndRendererFields() async throws {256 let api = Live.api()257 let options = try #require(try await api.lab.catalog().tool("options_lab"))258 var form = VQLabForm(tool: options)259 form["mode"] = "implied"260 guard case .failure(let invalid) = form.validate() else { Issue.record("the form accepted an implied-volatility request without a price"); return }261 do { _ = try await api.lab.start(tool: "options_lab", params: ["symbol": "SPY", "mode": "implied"]); Issue.record("the server accepted it") }262 catch let e as VQError { #expect(e.kind == .rejected && e.message == invalid.issues.first?.message, "server: \(e.message) — form: \(invalid.message)") }263 form["mode"] = "price"264 guard case .success(let params) = form.validate() else { Issue.record("defaults invalid"); return }265 let job = try await api.lab.run(tool: "options_lab", params: params, every: 0.5)266 let result = try #require(job.labResult)267 #expect(result.summary.filter { $0.unit == .currency }.allSatisfy { $0.currency != nil } && result.tables.flatMap(\.columns).allSatisfy { $0.type != nil })268 #expect(result.charts.allSatisfy { $0.y.unit != .currency || $0.y.currency != nil })269 try? await api.lab.delete(job.id)270 #expect(await api.researchAgent.researchModeAvailability() != nil) // the sandbox route answers: offer the research mode271 #expect(try await api.models.usage(days: 30).byModel.allSatisfy { $0.role != nil })272 }273274 @Test func sandboxStatusIsServed() async throws {275 let sandbox = try await Live.api().researchAgent.sandbox()276 #expect(sandbox.uploadMaxBytes > 0 && !sandbox.limits.isEmpty && !sandbox.defaultNetwork.isEmpty)277 #expect(sandbox.enforced || sandbox.note != nil)278 _ = try await Live.api().researchAgent.approvals()279 }280}281282/// The research agent's REST against a backend that holds a real study (the 5059 copy):283///284/// ssh -f -N -L 5059:localhost:5059 M2U64285/// VQ_LIVE=1 VQ_RESEARCH_BASE_URL=http://localhost:5059 VQ_RESEARCH_TOKEN="$(ssh M2U64 cat /tmp/vq-api-token-5059)" swift test --filter VQuantKitResearchLiveTests286///287/// Read-only: it lists sessions, reads a workspace tree, a file, the ledger, runs and logs. It never starts a study.288@Suite(.enabled(if: Live.env["VQ_LIVE"] == "1" && (Live.env["VQ_RESEARCH_TOKEN"] ?? "").hasPrefix("vqn_")), .serialized) struct VQuantKitResearchLiveTests {289 private func client() -> VQResearchAgentClient {290 let token = Live.env["VQ_RESEARCH_TOKEN"]291 let t = VQURLSessionTransport(baseURL: URL(string: Live.env["VQ_RESEARCH_BASE_URL"] ?? "http://localhost:5059")!, clientHeader: VQuantAPI.macClientHeader(version: "2.0.0"), token: { token })292 return VQuantAPI(transport: t).researchAgent293 }294295 @Test func sessionsTreeFileAndLedger() async throws {296 let c = client()297 let sessions = try await c.sessions()298 // the study with the most work in it299 var best: (VQResearchSession, VQResearchSessionDetail)?300 for s in sessions.prefix(8) {301 let d = try await c.session(s.id)302 if d.executions.count > (best?.1.executions.count ?? -1) { best = (s, d) }303 }304 let (session, detail) = try #require(best, "no research session on this backend")305 #expect(!detail.files.isEmpty && detail.session.id == session.id && detail.environment.hash != nil)306307 // the same workspace by conversation308 if let sid = session.sessionIds.first { #expect(try await c.session(VQResearchSession.reference(sessionId: sid)).session.id == session.id) }309310 let files = try await c.files(session.id)311 #expect(files.count == detail.files.count && !VQResearchFileTree.build(files).isEmpty)312 let code = try #require(files.first { !$0.isDirectory && $0.path.hasSuffix(".py") })313 let preview = try await c.textFile(session.id, path: code.path, offset: 0, limit: 5)314 #expect(preview.path == code.path && preview.to <= 5 && !preview.text.isEmpty && preview.totalLines >= preview.to)315 let bytes = try await c.fileData(session.id, path: code.path)316 #expect(bytes.count == code.bytes)317 if let png = files.first(where: { $0.path.hasSuffix(".png") }) {318 let data = try await c.fileData(session.id, path: png.path)319 #expect(data.prefix(4) == Data([0x89, 0x50, 0x4E, 0x47]))320 }321 do { _ = try await c.textFile(session.id, path: "no/such/file.py"); Issue.record("a missing file was served") }322 catch let e as VQError { #expect(e.kind == .notFound) }323324 let experiments = try await c.experiments(session.id, kind: .experiment)325 let executions = try await c.experiments(session.id, kind: .execution)326 #expect(experiments.count == detail.experiments.count && executions.count == detail.executions.count)327 #expect(experiments.allSatisfy { $0.kind == .experiment && $0.status.known != nil } && executions.allSatisfy { $0.kind == .execution })328 #expect(zip(executions, executions.dropFirst()).allSatisfy { ($0.seq ?? 0) < ($1.seq ?? 0) })329 let runs = try await c.runs(session.id)330 #expect(runs.count == detail.runs.count)331 if let run = runs.first {332 let logs = try await c.run(run.id, tail: 2000)333 #expect(logs.run.id == run.id && logs.stdout.count <= 2100)334 }335 if let data = files.first(where: { $0.path.hasPrefix("data/") && !$0.isDirectory }) {336 let inspection = try await c.inspect(session.id, path: data.path)337 #expect(inspection.path == data.path && !inspection.columns.isEmpty && (inspection.rows ?? 0) > 0)338 }339 // 0.3: generations with their plan, and — on studies recent enough — the figures with their verdict340 #expect(!detail.generations.isEmpty && detail.generations.allSatisfy { $0.status.known != nil && !$0.generationId.isEmpty })341 var sawFigureRecord = false, sawPlan = false342 for s in sessions.prefix(8) {343 let d = try await c.session(s.id)344 if !d.plan.isEmpty { sawPlan = true; #expect(VQResearchState(detail: d).plan == d.plan) }345 for f in d.figures {346 sawFigureRecord = true347 #expect(f.status?.known != nil && f.files?.png != nil)348 let rebuilt = try #require(VQResearchState(detail: d).figures.first { $0.figureId == f.id })349 #expect(rebuilt.lintStatus == f.status && rebuilt.caption == f.caption)350 if let png = f.files?.png { #expect(try await c.fileData(s.id, path: png).prefix(4) == Data([0x89, 0x50, 0x4E, 0x47])) }351 }352 }353 #expect(sawPlan && sawFigureRecord)354 // answered approvals are listed with their workspace355 let approvals = try await c.approvals(pendingOnly: false)356 #expect(approvals.allSatisfy { $0.workspaceId != nil && $0.status.known != nil })357 if let answered = approvals.first(where: { $0.status == .approved }), let ws = answered.workspaceId {358 let d = try await c.session(ws)359 #expect(VQResearchState(detail: d, approvals: approvals).approvals.contains { $0.id == answered.id })360 do { _ = try await c.answer(approval: answered.id, .deny); Issue.record("an answered approval was answered again") }361 catch let e as VQError { #expect(e.kind == .notFound) }362 }363 // the study rebuilt from REST alone364 let state = VQResearchState(detail: detail)365 #expect(state.counts.executions == detail.executions.count && state.experiments.count == detail.experiments.count && state.finished)366 #expect(try await c.sandbox().uploadMaxBytes > 0)367 do { _ = try await c.session("s-does-not-exist"); Issue.record("an unknown workspace was served") }368 catch let e as VQError { #expect(e.kind == .notFound) }369 }370}371