Repository Public
1import Foundation2import Testing3@testable import VQuantKit4import VQuantKitTesting56@Suite struct LabFormTests {7 private func catalog() throws -> VQLabCatalog { VQLabCatalog(json: try Fixture.json("quant_tools.json")) }8 private func params(_ form: VQLabForm) throws -> VQJSONObject {9 switch form.validate() {10 case .success(let p): return p11 case .failure(let invalid): Issue.record("unexpected issues: \(invalid.message)"); return [:]12 }13 }14 private func issues(_ form: VQLabForm) -> [String: String] {15 if case .failure(let invalid) = form.validate() { return Dictionary(invalid.issues.map { ($0.name, $0.message) }, uniquingKeysWith: { a, _ in a }) }16 return [:]17 }1819 /// The input the client builds equals the input the server recorded for the same request — for every recorded job.20 @Test(arguments: [21 ("volatility_lab", "volatility_lab", #"{"symbol":"AAPL","compare":true}"#),22 ("risk_lab", "risk_lab", #"{"symbols":["AAPL","MSFT","TLT"]}"#),23 ("regression", "regression", #"{"symbol":"AAPL","regressors":["SPY"]}"#),24 ("time_series", "time_series", #"{"symbol":"AAPL"}"#),25 ("options_strategy", "options_lab", #"{"symbol":"SPY","mode":"strategy","template":"straddle"}"#),26 ("options_chain", "options_lab", #"{"symbol":"SPY","mode":"chain"}"#),27 ("backtest", "backtest", #"{"symbols":["SPY"],"sensitivity":true}"#),28 ("portfolio_lab", "portfolio_lab", #"{"symbols":["SPY","TLT","GLD"]}"#),29 ("var_model", "var_model", #"{"symbols":["SPY","TLT"]}"#),30 ("pca", "pca", #"{"symbols":["AAPL","MSFT","NVDA","GOOGL","AMZN","JPM"]}"#),31 ("factor_model", "factor_model", #"{"symbols":["AAPL"]}"#),32 ("bootstrap", "bootstrap", #"{"symbol":"AAPL"}"#),33 ("panel", "panel", #"{"symbols":["AAPL","MSFT","NVDA","GOOGL","AMZN","JPM"]}"#),34 ("event_study", "event_study", #"{"symbols":["AAPL"]}"#),35 ("legacy_garch", "garch", #"{"symbol":"AAPL"}"#),36 ])37 func buildsWhatTheServerRecorded(fixture: String, tool: String, input: String) throws {38 let tool = try #require(try catalog().tool(tool))39 var form = VQLabForm(tool: tool)40 for (k, v) in try VQJSON.parse(input).objectValue! { form[k] = v }41 let recorded = try Fixture.json("job_\(fixture).json")["job"]!.dict("params")42 #expect(try params(form) == recorded)43 }4445 @Test func requiredAndDefaults() throws {46 let tool = try #require(try catalog().tool("volatility_lab"))47 var form = VQLabForm(tool: tool)48 #expect(issues(form) == ["symbol": "Symbol is required."])49 #expect(!form.isValid && form.issue(for: "symbol") != nil && form.issue(for: "p") == nil)50 form["symbol"] = " aapl "51 let p = try params(form)52 #expect(p["symbol"] == "AAPL" && p["horizon"] == 21 && p["model"] == "garch" && p["har"] == true)53 #expect(p["start"] == nil) // optional without default: omitted54 form["unknown_key"] = "dropped"55 #expect(try params(form)["unknown_key"] == nil)56 form.reset()57 #expect(form["symbol"] == nil)58 }5960 @Test func everyParameterTypeIsCheckedLikeTheServer() throws {61 let c = try catalog()62 // symbol / symbols63 var risk = VQLabForm(tool: try #require(c.tool("risk_lab")))64 risk["symbols"] = "aapl, msft; tlt aapl" // typed text: separators, duplicates, case65 #expect(try params(risk)["symbols"] == ["AAPL", "MSFT", "TLT"])66 risk["symbols"] = "AAPL, not a symbol!"67 #expect(issues(risk)["symbols"]?.contains("is not a ticker symbol") == true)68 risk["symbols"] = .array((1...40).map { .string("S\($0)") })69 #expect(issues(risk)["symbols"]?.contains("distinct symbols") == true)70 risk["symbols"] = ["^GSPC", "BRK-B", "ES=F"]71 #expect(try params(risk)["symbols"] == ["^GSPC", "BRK-B", "ES=F"])72 // numbers73 risk["confidence_levels"] = "0.9 0.95; 0.99" // in a list the comma separates, as on the server74 #expect(try params(risk)["confidence_levels"] == [0.9, 0.95, 0.99])75 risk["confidence_levels"] = "0.95, 7"76 #expect(issues(risk)["confidence_levels"]?.contains("must be between") == true)77 risk["confidence_levels"] = "abc"78 #expect(issues(risk)["confidence_levels"] == "Confidence levels: a number is expected.")79 risk["confidence_levels"] = [0.95, 0.99]80 // int / number81 risk["simulations"] = "20000.5"82 #expect(issues(risk)["simulations"]?.hasSuffix("an integer is expected.") == true)83 risk["simulations"] = "5000"84 #expect(try params(risk)["simulations"] == 5000)85 risk["simulations"] = 186 #expect(issues(risk)["simulations"]?.contains("must be between") == true)87 risk["simulations"] = 500088 risk["corr_lambda"] = "0,97" // a decimal comma from a fr-CA keyboard89 #expect(try params(risk)["corr_lambda"] == 0.97)90 // choice91 risk["fhs_model"] = "magic"92 #expect(issues(risk)["fhs_model"]?.contains("one of") == true)93 risk["fhs_model"] = "garch"94 // json: text or structure, object/array only, size and depth95 risk["shocks"] = #"[{"name":"crash","AAPL":-0.2}]"#96 #expect(try params(risk)["shocks"] == [["name": "crash", "AAPL": -0.2]])97 risk["shocks"] = "{not json"98 #expect(issues(risk)["shocks"] == "Scenario shocks: this is not valid JSON." || issues(risk)["shocks"]?.hasSuffix("this is not valid JSON.") == true)99 risk["shocks"] = "42"100 #expect(issues(risk)["shocks"]?.hasSuffix("a JSON object or array is expected.") == true)101 var deep: VQJSON = [1]102 for _ in 0..<13 { deep = .array([deep]) }103 risk["shocks"] = deep104 #expect(issues(risk)["shocks"]?.hasSuffix("nested too deeply.") == true)105 risk["shocks"] = .array(Array(repeating: .string(String(repeating: "x", count: 1000)), count: 80))106 #expect(issues(risk)["shocks"]?.contains("too large") == true)107 risk["shocks"] = nil108 // date109 risk["start"] = "2024-02-30"110 #expect(issues(risk)["start"]?.hasSuffix("a date YYYY-MM-DD is expected.") == true)111 risk["start"] = "20 nov 2024"112 #expect(issues(risk)["start"] != nil)113 risk["start"] = " 2024-02-29 "114 #expect(try params(risk)["start"] == "2024-02-29")115 // bool116 var vol = VQLabForm(tool: try #require(c.tool("volatility_lab")), values: ["symbol": "SPY"])117 vol["compare"] = "true"118 #expect(try params(vol)["compare"] == true && vol.bool("compare"))119 vol["compare"] = "yes"120 #expect(try params(vol)["compare"] == false)121 // text: control characters removed, trimmed, length, empty falls back to the default122 vol["compare_specs"] = " ewma,\u{0007} garch:t \n"123 #expect(try params(vol)["compare_specs"] == "ewma, garch:t")124 vol["compare_specs"] = .string(String(repeating: "g", count: 2001))125 #expect(issues(vol)["compare_specs"]?.contains("at most 2000 characters") == true)126 vol["compare_specs"] = " "127 #expect(try params(vol)["compare_specs"] == "ewma, garch:normal, garch:t, gjr:t, egarch:t")128 // all issues are reported together129 vol["symbol"] = ""; vol["horizon"] = "x"130 #expect(Set(issues(vol).keys) == ["symbol", "horizon"])131 }132133 @Test func dateBoundsCrossFieldRuleAndFutureTypes() throws {134 let tool = VQLabTool(json: ["id": "t", "title": "T", "resultSchema": "lab/1", "params": [135 ["name": "heading", "label": "Window", "type": "group"],136 ["name": "from", "label": "From", "type": "date", "minDate": "2020-01-01", "maxDate": "2026-12-31", "help": "Start of the sample."],137 ["name": "min_weight", "label": "Min", "type": "number", "min": 0, "max": 1, "default": 0],138 ["name": "max_weight", "label": "Max", "type": "number", "min": 0, "max": 1, "default": 1],139 ["name": "picker", "label": "Colour", "type": "colour", "default": "red"],140 ]])141 #expect(tool.params[0].type == .group && tool.params[4].type.known == nil)142 #expect(tool.groups.map(\.name) == ["Window"] && tool.groups[0].params.count == 4) // the heading groups what follows143 var form = VQLabForm(tool: tool)144 form["from"] = "2019-12-31"145 #expect(issues(form)["from"] == "From: not before 2020-01-01.")146 form["from"] = "2027-01-01"147 #expect(issues(form)["from"] == "From: not after 2026-12-31.")148 form["from"] = "2024-06-30"; form["min_weight"] = 0.8; form["max_weight"] = 0.2149 #expect(issues(form)[""] == "Minimum weight is above maximum weight.")150 form["min_weight"] = 0.1151 let p = try params(form)152 #expect(p["picker"] == "red" && p["heading"] == nil && p["from"] == "2024-06-30") // an unknown type is sent as text153 #expect(form.text("min_weight") == "0.1" && form.text("from") == "2024-06-30" && form.text("nothing") == "")154 }155156 @Test func legacyNumbersDefaultToTenItems() throws {157 let tool = try #require(try catalog().tool("var"))158 var form = VQLabForm(tool: tool, values: ["symbol": "AAPL"])159 #expect(form.text("confidence_levels") == "0.9, 0.95, 0.99")160 form["confidence_levels"] = .array((0..<11).map { _ in 0.95 })161 #expect(issues(form)["confidence_levels"]?.hasSuffix("1 to 10 values.") == true)162 }163}164165@Suite struct LabJobTests {166 private func job(_ status: String, progress: Double?, result: VQJSON = .null) -> VQJSON {167 ["job": ["id": "j1", "tool": "risk_lab", "params": ["symbols": ["AAPL"]], "status": .string(status), "startedAt": 1_795_000_000_000,168 "finishedAt": status == "running" ? .null : 1_795_000_004_000, "progress": VQJSON(progress), "durationMs": status == "running" ? .null : 4000,169 "cancellable": .bool(status == "running"), "result": result]]170 }171172 @Test func startPollWithProgressUntilDone() async throws {173 let done = try Fixture.json("job_risk_lab.json")174 let transport = VQScriptedTransport([175 .init(status: 202, json: job("running", progress: 0)),176 .init(json: job("running", progress: 0)),177 .init(json: job("running", progress: 0)), // unchanged: not yielded again178 .init(status: 503, json: ["error": "restarting"]), // a transient failure is retried179 .init(json: job("running", progress: 0.4)),180 .init(json: job("running", progress: 0.85)),181 .init(json: done),182 ])183 let clock = VQManualClock()184 let lab = VQLabClient(client: VQClient(transport: transport), clock: clock)185 let started = try await lab.start(tool: "risk_lab", params: ["symbols": ["AAPL"]], projectId: "p1")186 #expect(started.isRunning && started.cancellable)187 let body = try VQJSON.parse(transport.requests[0].body!)188 #expect(body == ["tool": "risk_lab", "params": ["symbols": ["AAPL"]], "projectId": "p1"] && transport.requests[0].path == "/api/quant/jobs")189190 let pump = Task { while !Task.isCancelled { await clock.waitForSleepers(1, timeout: 1); clock.advance(by: 1) } }191 var seen: [Double?] = [], final: VQQuantJob?192 for try await state in lab.follow(started.id, every: 1) { seen.append(state.progress); final = state }193 pump.cancel()194 #expect(seen == [0, 0.4, 0.85, 1])195 #expect(final?.status == .done && final?.labResult?.engine == "risk_lab" && final?.labResult?.charts.isEmpty == false)196 #expect(clock.recordedSleeps.count == 5 && clock.recordedSleeps.allSatisfy { $0 == 1 })197 #expect(transport.requests.dropFirst().allSatisfy { $0.path == "/api/quant/jobs/j1" && $0.method == .get })198 }199200 @Test func cancelOutcomes() async throws {201 let transport = VQScriptedTransport([202 .init(json: try Fixture.json("job_cancel.json")),203 .init(status: 409, json: try Fixture.json("job_cancel_not_running.json")),204 .init(status: 409, json: ["error": "not_cancellable"]),205 .init(status: 404, json: ["error": "Computation not found."]),206 ])207 let lab = VQLabClient(client: VQClient(transport: transport))208 guard case .cancelled(let job) = try await lab.cancel("j1") else { Issue.record("expected cancelled"); return }209 #expect(job.status == .cancelled && job.result == nil && !job.cancellable)210 #expect(try await lab.cancel("j1") == .notRunning(status: "cancelled"))211 #expect(try await lab.cancel("legacy") == .notCancellable)212 await #expect(throws: VQError.self) { try await lab.cancel("missing") }213 #expect(transport.requests.allSatisfy { $0.method == .post && $0.path.hasSuffix("/cancel") })214 }215216 @Test func followStopsOnPermanentFailureAndAfterTooManyTransientOnes() async throws {217 let clock = VQManualClock()218 let gone = VQLabClient(client: VQClient(transport: VQScriptedTransport([.init(status: 404, json: ["error": "Computation not found."])])), clock: clock)219 await #expect(throws: VQError.self) { for try await _ in gone.follow("x") {} }220 let flaky = VQScriptedTransport((0..<3).map { _ in .init(status: 503, json: ["error": "down"]) })221 let lab = VQLabClient(client: VQClient(transport: flaky), clock: clock)222 let pump = Task { while !Task.isCancelled { await clock.waitForSleepers(1, timeout: 1); clock.advance(by: 1) } }223 await #expect(throws: VQError.self) { for try await _ in lab.follow("x", every: 1, maxConsecutiveFailures: 2) {} }224 pump.cancel()225 #expect(flaky.requests.count == 3)226 }227228 @Test func runReturnsTheFinalState() async throws {229 let transport = VQScriptedTransport([230 .init(status: 202, json: job("running", progress: nil)),231 .init(json: job("cancelled", progress: 0)),232 ])233 let clock = VQManualClock()234 let lab = VQLabClient(client: VQClient(transport: transport), clock: clock)235 let final = try await lab.run(tool: "risk_lab", params: [:], every: 1)236 #expect(final.status == .cancelled && final.labResult == nil)237 }238239 @Test func researchRequests() async throws {240 let transport = VQScriptedTransport([241 .init(status: 202, json: ["generationId": "g1", "sessionId": "s1"]),242 .init(json: ["filename": "f.zip", "url": "/api/download/f.zip", "format": "zip", "bytes": 10, "items": 2, "compiled": false]),243 .init(status: 202, json: ["reviewId": "r1"]),244 .init(json: ["deleted": 3]),245 .init(json: ["settings": ["enabled": false, "retentionDays": 90]]),246 ])247 let r = VQResearchClient(client: VQClient(transport: transport))248 let run = try await r.run(project: "p 1", prompt: "Go", mode: "deep", model: "auto", budget: VQBudget(maxCostUsd: 2))249 #expect(run.generationId == "g1" && run.sessionId == "s1")250 #expect(transport.requests[0].path == "/api/v2/projects/p%201/run")251 #expect(try VQJSON.parse(transport.requests[0].body!) == ["prompt": "Go", "mode": "deep", "model": "auto", "budget": ["maxCostUsd": 2]])252 let export = try await r.export(project: "p1", format: .zip, include: ["it1"])253 #expect(export.compiled == false && export.url == "/api/download/f.zip")254 #expect(try await r.startReview(of: .init(type: .quantJob, id: "j1"), projectId: "p1") == "r1")255 #expect(try VQJSON.parse(transport.requests[2].body!) == ["target": ["type": "quant_job", "id": "j1"], "projectId": "p1"])256 #expect(try await r.deleteMemories(scope: .project("p1")) == 3)257 #expect(transport.requests[3].query == [VQQueryItem("scope", "project:p1")])258 let s = try await r.setMemorySettings(enabled: false, retentionDays: .some(90))259 #expect(s == VQMemorySettings(enabled: false, retentionDays: 90))260 #expect(VQMemoryChange(pinned: true, expiresAt: .some(nil)).json == ["pinned": true, "expiresAt": .null])261 }262}263