SPB Git

Repository Public

LabTests.swift 15.8 KB 263 lines swift

1import Foundation2import Testing3@testable import VQuantKit4import VQuantKitTesting56@Suite struct LabFormTests {7    private func catalog() throws -> VQLabCatalog { VQLabCatalog(json: try Fixture.json("quant_tools.json")) }8    private func params(_ form: VQLabForm) throws -> VQJSONObject {9        switch form.validate() {10        case .success(let p): return p11        case .failure(let invalid): Issue.record("unexpected issues: \(invalid.message)"); return [:]12        }13    }14    private func issues(_ form: VQLabForm) -> [String: String] {15        if case .failure(let invalid) = form.validate() { return Dictionary(invalid.issues.map { ($0.name, $0.message) }, uniquingKeysWith: { a, _ in a }) }16        return [:]17    }1819    /// The input the client builds equals the input the server recorded for the same request — for every recorded job.20    @Test(arguments: [21        ("volatility_lab", "volatility_lab", #"{"symbol":"AAPL","compare":true}"#),22        ("risk_lab", "risk_lab", #"{"symbols":["AAPL","MSFT","TLT"]}"#),23        ("regression", "regression", #"{"symbol":"AAPL","regressors":["SPY"]}"#),24        ("time_series", "time_series", #"{"symbol":"AAPL"}"#),25        ("options_strategy", "options_lab", #"{"symbol":"SPY","mode":"strategy","template":"straddle"}"#),26        ("options_chain", "options_lab", #"{"symbol":"SPY","mode":"chain"}"#),27        ("backtest", "backtest", #"{"symbols":["SPY"],"sensitivity":true}"#),28        ("portfolio_lab", "portfolio_lab", #"{"symbols":["SPY","TLT","GLD"]}"#),29        ("var_model", "var_model", #"{"symbols":["SPY","TLT"]}"#),30        ("pca", "pca", #"{"symbols":["AAPL","MSFT","NVDA","GOOGL","AMZN","JPM"]}"#),31        ("factor_model", "factor_model", #"{"symbols":["AAPL"]}"#),32        ("bootstrap", "bootstrap", #"{"symbol":"AAPL"}"#),33        ("panel", "panel", #"{"symbols":["AAPL","MSFT","NVDA","GOOGL","AMZN","JPM"]}"#),34        ("event_study", "event_study", #"{"symbols":["AAPL"]}"#),35        ("legacy_garch", "garch", #"{"symbol":"AAPL"}"#),36    ])37    func buildsWhatTheServerRecorded(fixture: String, tool: String, input: String) throws {38        let tool = try #require(try catalog().tool(tool))39        var form = VQLabForm(tool: tool)40        for (k, v) in try VQJSON.parse(input).objectValue! { form[k] = v }41        let recorded = try Fixture.json("job_\(fixture).json")["job"]!.dict("params")42        #expect(try params(form) == recorded)43    }4445    @Test func requiredAndDefaults() throws {46        let tool = try #require(try catalog().tool("volatility_lab"))47        var form = VQLabForm(tool: tool)48        #expect(issues(form) == ["symbol": "Symbol is required."])49        #expect(!form.isValid && form.issue(for: "symbol") != nil && form.issue(for: "p") == nil)50        form["symbol"] = " aapl "51        let p = try params(form)52        #expect(p["symbol"] == "AAPL" && p["horizon"] == 21 && p["model"] == "garch" && p["har"] == true)53        #expect(p["start"] == nil)                                             // optional without default: omitted54        form["unknown_key"] = "dropped"55        #expect(try params(form)["unknown_key"] == nil)56        form.reset()57        #expect(form["symbol"] == nil)58    }5960    @Test func everyParameterTypeIsCheckedLikeTheServer() throws {61        let c = try catalog()62        // symbol / symbols63        var risk = VQLabForm(tool: try #require(c.tool("risk_lab")))64        risk["symbols"] = "aapl, msft;  tlt aapl"                               // typed text: separators, duplicates, case65        #expect(try params(risk)["symbols"] == ["AAPL", "MSFT", "TLT"])66        risk["symbols"] = "AAPL, not a symbol!"67        #expect(issues(risk)["symbols"]?.contains("is not a ticker symbol") == true)68        risk["symbols"] = .array((1...40).map { .string("S\($0)") })69        #expect(issues(risk)["symbols"]?.contains("distinct symbols") == true)70        risk["symbols"] = ["^GSPC", "BRK-B", "ES=F"]71        #expect(try params(risk)["symbols"] == ["^GSPC", "BRK-B", "ES=F"])72        // numbers73        risk["confidence_levels"] = "0.9 0.95; 0.99"                             // in a list the comma separates, as on the server74        #expect(try params(risk)["confidence_levels"] == [0.9, 0.95, 0.99])75        risk["confidence_levels"] = "0.95, 7"76        #expect(issues(risk)["confidence_levels"]?.contains("must be between") == true)77        risk["confidence_levels"] = "abc"78        #expect(issues(risk)["confidence_levels"] == "Confidence levels: a number is expected.")79        risk["confidence_levels"] = [0.95, 0.99]80        // int / number81        risk["simulations"] = "20000.5"82        #expect(issues(risk)["simulations"]?.hasSuffix("an integer is expected.") == true)83        risk["simulations"] = "5000"84        #expect(try params(risk)["simulations"] == 5000)85        risk["simulations"] = 186        #expect(issues(risk)["simulations"]?.contains("must be between") == true)87        risk["simulations"] = 500088        risk["corr_lambda"] = "0,97"                                            // a decimal comma from a fr-CA keyboard89        #expect(try params(risk)["corr_lambda"] == 0.97)90        // choice91        risk["fhs_model"] = "magic"92        #expect(issues(risk)["fhs_model"]?.contains("one of") == true)93        risk["fhs_model"] = "garch"94        // json: text or structure, object/array only, size and depth95        risk["shocks"] = #"[{"name":"crash","AAPL":-0.2}]"#96        #expect(try params(risk)["shocks"] == [["name": "crash", "AAPL": -0.2]])97        risk["shocks"] = "{not json"98        #expect(issues(risk)["shocks"] == "Scenario shocks: this is not valid JSON." || issues(risk)["shocks"]?.hasSuffix("this is not valid JSON.") == true)99        risk["shocks"] = "42"100        #expect(issues(risk)["shocks"]?.hasSuffix("a JSON object or array is expected.") == true)101        var deep: VQJSON = [1]102        for _ in 0..<13 { deep = .array([deep]) }103        risk["shocks"] = deep104        #expect(issues(risk)["shocks"]?.hasSuffix("nested too deeply.") == true)105        risk["shocks"] = .array(Array(repeating: .string(String(repeating: "x", count: 1000)), count: 80))106        #expect(issues(risk)["shocks"]?.contains("too large") == true)107        risk["shocks"] = nil108        // date109        risk["start"] = "2024-02-30"110        #expect(issues(risk)["start"]?.hasSuffix("a date YYYY-MM-DD is expected.") == true)111        risk["start"] = "20 nov 2024"112        #expect(issues(risk)["start"] != nil)113        risk["start"] = " 2024-02-29 "114        #expect(try params(risk)["start"] == "2024-02-29")115        // bool116        var vol = VQLabForm(tool: try #require(c.tool("volatility_lab")), values: ["symbol": "SPY"])117        vol["compare"] = "true"118        #expect(try params(vol)["compare"] == true && vol.bool("compare"))119        vol["compare"] = "yes"120        #expect(try params(vol)["compare"] == false)121        // text: control characters removed, trimmed, length, empty falls back to the default122        vol["compare_specs"] = "  ewma,\u{0007} garch:t \n"123        #expect(try params(vol)["compare_specs"] == "ewma, garch:t")124        vol["compare_specs"] = .string(String(repeating: "g", count: 2001))125        #expect(issues(vol)["compare_specs"]?.contains("at most 2000 characters") == true)126        vol["compare_specs"] = "   "127        #expect(try params(vol)["compare_specs"] == "ewma, garch:normal, garch:t, gjr:t, egarch:t")128        // all issues are reported together129        vol["symbol"] = ""; vol["horizon"] = "x"130        #expect(Set(issues(vol).keys) == ["symbol", "horizon"])131    }132133    @Test func dateBoundsCrossFieldRuleAndFutureTypes() throws {134        let tool = VQLabTool(json: ["id": "t", "title": "T", "resultSchema": "lab/1", "params": [135            ["name": "heading", "label": "Window", "type": "group"],136            ["name": "from", "label": "From", "type": "date", "minDate": "2020-01-01", "maxDate": "2026-12-31", "help": "Start of the sample."],137            ["name": "min_weight", "label": "Min", "type": "number", "min": 0, "max": 1, "default": 0],138            ["name": "max_weight", "label": "Max", "type": "number", "min": 0, "max": 1, "default": 1],139            ["name": "picker", "label": "Colour", "type": "colour", "default": "red"],140        ]])141        #expect(tool.params[0].type == .group && tool.params[4].type.known == nil)142        #expect(tool.groups.map(\.name) == ["Window"] && tool.groups[0].params.count == 4)       // the heading groups what follows143        var form = VQLabForm(tool: tool)144        form["from"] = "2019-12-31"145        #expect(issues(form)["from"] == "From: not before 2020-01-01.")146        form["from"] = "2027-01-01"147        #expect(issues(form)["from"] == "From: not after 2026-12-31.")148        form["from"] = "2024-06-30"; form["min_weight"] = 0.8; form["max_weight"] = 0.2149        #expect(issues(form)[""] == "Minimum weight is above maximum weight.")150        form["min_weight"] = 0.1151        let p = try params(form)152        #expect(p["picker"] == "red" && p["heading"] == nil && p["from"] == "2024-06-30")        // an unknown type is sent as text153        #expect(form.text("min_weight") == "0.1" && form.text("from") == "2024-06-30" && form.text("nothing") == "")154    }155156    @Test func legacyNumbersDefaultToTenItems() throws {157        let tool = try #require(try catalog().tool("var"))158        var form = VQLabForm(tool: tool, values: ["symbol": "AAPL"])159        #expect(form.text("confidence_levels") == "0.9, 0.95, 0.99")160        form["confidence_levels"] = .array((0..<11).map { _ in 0.95 })161        #expect(issues(form)["confidence_levels"]?.hasSuffix("1 to 10 values.") == true)162    }163}164165@Suite struct LabJobTests {166    private func job(_ status: String, progress: Double?, result: VQJSON = .null) -> VQJSON {167        ["job": ["id": "j1", "tool": "risk_lab", "params": ["symbols": ["AAPL"]], "status": .string(status), "startedAt": 1_795_000_000_000,168                 "finishedAt": status == "running" ? .null : 1_795_000_004_000, "progress": VQJSON(progress), "durationMs": status == "running" ? .null : 4000,169                 "cancellable": .bool(status == "running"), "result": result]]170    }171172    @Test func startPollWithProgressUntilDone() async throws {173        let done = try Fixture.json("job_risk_lab.json")174        let transport = VQScriptedTransport([175            .init(status: 202, json: job("running", progress: 0)),176            .init(json: job("running", progress: 0)),177            .init(json: job("running", progress: 0)),                           // unchanged: not yielded again178            .init(status: 503, json: ["error": "restarting"]),                  // a transient failure is retried179            .init(json: job("running", progress: 0.4)),180            .init(json: job("running", progress: 0.85)),181            .init(json: done),182        ])183        let clock = VQManualClock()184        let lab = VQLabClient(client: VQClient(transport: transport), clock: clock)185        let started = try await lab.start(tool: "risk_lab", params: ["symbols": ["AAPL"]], projectId: "p1")186        #expect(started.isRunning && started.cancellable)187        let body = try VQJSON.parse(transport.requests[0].body!)188        #expect(body == ["tool": "risk_lab", "params": ["symbols": ["AAPL"]], "projectId": "p1"] && transport.requests[0].path == "/api/quant/jobs")189190        let pump = Task { while !Task.isCancelled { await clock.waitForSleepers(1, timeout: 1); clock.advance(by: 1) } }191        var seen: [Double?] = [], final: VQQuantJob?192        for try await state in lab.follow(started.id, every: 1) { seen.append(state.progress); final = state }193        pump.cancel()194        #expect(seen == [0, 0.4, 0.85, 1])195        #expect(final?.status == .done && final?.labResult?.engine == "risk_lab" && final?.labResult?.charts.isEmpty == false)196        #expect(clock.recordedSleeps.count == 5 && clock.recordedSleeps.allSatisfy { $0 == 1 })197        #expect(transport.requests.dropFirst().allSatisfy { $0.path == "/api/quant/jobs/j1" && $0.method == .get })198    }199200    @Test func cancelOutcomes() async throws {201        let transport = VQScriptedTransport([202            .init(json: try Fixture.json("job_cancel.json")),203            .init(status: 409, json: try Fixture.json("job_cancel_not_running.json")),204            .init(status: 409, json: ["error": "not_cancellable"]),205            .init(status: 404, json: ["error": "Computation not found."]),206        ])207        let lab = VQLabClient(client: VQClient(transport: transport))208        guard case .cancelled(let job) = try await lab.cancel("j1") else { Issue.record("expected cancelled"); return }209        #expect(job.status == .cancelled && job.result == nil && !job.cancellable)210        #expect(try await lab.cancel("j1") == .notRunning(status: "cancelled"))211        #expect(try await lab.cancel("legacy") == .notCancellable)212        await #expect(throws: VQError.self) { try await lab.cancel("missing") }213        #expect(transport.requests.allSatisfy { $0.method == .post && $0.path.hasSuffix("/cancel") })214    }215216    @Test func followStopsOnPermanentFailureAndAfterTooManyTransientOnes() async throws {217        let clock = VQManualClock()218        let gone = VQLabClient(client: VQClient(transport: VQScriptedTransport([.init(status: 404, json: ["error": "Computation not found."])])), clock: clock)219        await #expect(throws: VQError.self) { for try await _ in gone.follow("x") {} }220        let flaky = VQScriptedTransport((0..<3).map { _ in .init(status: 503, json: ["error": "down"]) })221        let lab = VQLabClient(client: VQClient(transport: flaky), clock: clock)222        let pump = Task { while !Task.isCancelled { await clock.waitForSleepers(1, timeout: 1); clock.advance(by: 1) } }223        await #expect(throws: VQError.self) { for try await _ in lab.follow("x", every: 1, maxConsecutiveFailures: 2) {} }224        pump.cancel()225        #expect(flaky.requests.count == 3)226    }227228    @Test func runReturnsTheFinalState() async throws {229        let transport = VQScriptedTransport([230            .init(status: 202, json: job("running", progress: nil)),231            .init(json: job("cancelled", progress: 0)),232        ])233        let clock = VQManualClock()234        let lab = VQLabClient(client: VQClient(transport: transport), clock: clock)235        let final = try await lab.run(tool: "risk_lab", params: [:], every: 1)236        #expect(final.status == .cancelled && final.labResult == nil)237    }238239    @Test func researchRequests() async throws {240        let transport = VQScriptedTransport([241            .init(status: 202, json: ["generationId": "g1", "sessionId": "s1"]),242            .init(json: ["filename": "f.zip", "url": "/api/download/f.zip", "format": "zip", "bytes": 10, "items": 2, "compiled": false]),243            .init(status: 202, json: ["reviewId": "r1"]),244            .init(json: ["deleted": 3]),245            .init(json: ["settings": ["enabled": false, "retentionDays": 90]]),246        ])247        let r = VQResearchClient(client: VQClient(transport: transport))248        let run = try await r.run(project: "p 1", prompt: "Go", mode: "deep", model: "auto", budget: VQBudget(maxCostUsd: 2))249        #expect(run.generationId == "g1" && run.sessionId == "s1")250        #expect(transport.requests[0].path == "/api/v2/projects/p%201/run")251        #expect(try VQJSON.parse(transport.requests[0].body!) == ["prompt": "Go", "mode": "deep", "model": "auto", "budget": ["maxCostUsd": 2]])252        let export = try await r.export(project: "p1", format: .zip, include: ["it1"])253        #expect(export.compiled == false && export.url == "/api/download/f.zip")254        #expect(try await r.startReview(of: .init(type: .quantJob, id: "j1"), projectId: "p1") == "r1")255        #expect(try VQJSON.parse(transport.requests[2].body!) == ["target": ["type": "quant_job", "id": "j1"], "projectId": "p1"])256        #expect(try await r.deleteMemories(scope: .project("p1")) == 3)257        #expect(transport.requests[3].query == [VQQueryItem("scope", "project:p1")])258        let s = try await r.setMemorySettings(enabled: false, retentionDays: .some(90))259        #expect(s == VQMemorySettings(enabled: false, retentionDays: 90))260        #expect(VQMemoryChange(pinned: true, expiresAt: .some(nil)).json == ["pinned": true, "expiresAt": .null])261    }262}263
Files main
Menu

Theme