Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/UITests/ResearchLiveUITests.swift3// Description: LIVE verification of Research 2.0 from the composer, by clicks, on the test backend (5058). These tests CALL A MODEL and cost money: each one is skipped unless VQ_LIVE names it (`TEST_RUNNER_VQ_LIVE=fast|research`), so a normal run of the suite never spends anything.4import XCTest56final class ResearchLiveUITests: XCTestCase {7 private let shots = URL(fileURLWithPath: NSHomeDirectory()).appendingPathComponent("Library/Developer/VQuant/shots/research/live", isDirectory: true)89 private func launch(_ which: String, _ extra: [String]) throws -> XCUIApplication {10 let env = ProcessInfo.processInfo.environment11 guard env["VQ_LIVE"] == which else { throw XCTSkip("live test: set TEST_RUNNER_VQ_LIVE=\(which) to spend a model call") }12 guard let token = env["VQ_DEV_TOKEN"], let server = env["VQ_SERVER"], server.hasSuffix(":5058") else { throw XCTSkip("needs the 5058 development session") }13 let app = XCUIApplication()14 app.launchArguments = ["-VQUITest", "-VQInMemory", "-VQSection", "research"] + extra15 app.launchEnvironment = ["VQ_DEV_TOKEN": token, "VQ_SERVER": server]16 app.launch()17 XCTAssertTrue(app.windows.firstMatch.waitForExistence(timeout: 20))18 try? FileManager.default.createDirectory(at: shots, withIntermediateDirectories: true)19 return app20 }21 private func el(_ app: XCUIApplication, _ id: String) -> XCUIElement { app.descendants(matching: .any)[id].firstMatch }22 private func shot(_ app: XCUIApplication, _ name: String) {23 try? app.windows.firstMatch.screenshot().pngRepresentation.write(to: shots.appendingPathComponent(name + ".png"))24 }25 private func type(_ app: XCUIApplication, _ question: String) {26 let field = el(app, "composer.text")27 XCTAssertTrue(field.waitForExistence(timeout: 10))28 field.click(); field.typeText(question)29 }3031 /// ONE fast analysis with `auto`: the router's decision is shown, then the usage line under the answer.32 func testFastAnalysisWithAutoRouting() throws {33 let app = try launch("fast", ["-VQXMode", "fast", "-VQXModel", "auto"])34 type(app, "In one sentence: what is the last closing price of MSFT, with its date?")35 el(app, "composer.send").click()36 XCTAssertTrue(el(app, "route.line").waitForExistence(timeout: 60), "model_routed is shown (model and reason)")37 shot(app, "fast-routed")38 XCTAssertTrue(el(app, "usage.line").waitForExistence(timeout: 240), "the usage line appears when the analysis is done")39 shot(app, "fast-done")40 el(app, "usage.line").click(); sleep(1); shot(app, "fast-usage-popover")41 app.typeKey(.escape, modifierFlags: [])42 if el(app, "route.candidates").exists { el(app, "route.candidates").click(); sleep(1); shot(app, "fast-candidates"); app.typeKey(.escape, modifierFlags: []) }43 if el(app, "provenance.button").exists { el(app, "provenance.button").click(); sleep(1); shot(app, "fast-provenance"); app.typeKey(.escape, modifierFlags: []) }44 if el(app, "answer.trace").exists { el(app, "answer.trace").click(); XCTAssertTrue(el(app, "trace.done").waitForExistence(timeout: 15)); sleep(2); shot(app, "fast-trace"); el(app, "trace.done").click() }45 }4647 /// ONE research study, Sonnet 5.5, cost cap 3 USD, with the tiny CSV attached at launch (`-VQXDataset`).48 func testResearchStudyWithDataset() throws {49 let app = try launch("research", ["-VQXMode", "research", "-VQXModel", "claude-sonnet-5-5", "-VQXDataset", "~/tmp/mac-test-closes.csv"])50 XCTAssertTrue(el(app, "dataset.profile").waitForExistence(timeout: 60), "the dataset was uploaded and profiled")51 el(app, "composer.budget").click()52 let cost = el(app, "budget.cost")53 XCTAssertTrue(cost.waitForExistence(timeout: 5))54 cost.click(); cost.typeText("3")55 el(app, "budget.set").click()56 type(app, "Using the attached CSV of daily closes, estimate a GARCH(1,1) on log returns, report persistence and a 5-day volatility forecast, with one figure. Keep it short.")57 shot(app, "study-composer")58 el(app, "composer.send").click()59 XCTAssertTrue(el(app, "inspector.tab.activite").waitForExistence(timeout: 90), "the cockpit opens as soon as the study has a workspace")60 let tabs = ["activite", "code", "experiences", "figures", "fichiers", "environnement"]61 var round = 0, denied = false62 let deadline = Date().addingTimeInterval(40 * 60)63 while !el(app, "usage.line").exists, Date() < deadline {64 if !denied, el(app, "approval.deny").exists || el(app, "approval.review").exists {65 // the budget of this verification must not be exceeded and no network is needed: any request is refused66 shot(app, "study-approval")67 if !el(app, "approval.deny").exists { el(app, "approval.review").click(); _ = el(app, "approval.deny").waitForExistence(timeout: 5) }68 shot(app, "study-approval-sheet")69 el(app, "approval.deny").click(); denied = true70 }71 if round < 12 {72 let t = tabs[round % tabs.count]73 el(app, "inspector.tab.\(t)").click(); sleep(3)74 shot(app, String(format: "study-running-%02d-%@", round, t))75 }76 round += 177 sleep(20)78 }79 XCTAssertTrue(el(app, "usage.line").exists, "the study finished with a usage line")80 sleep(8) // the workspace is read again by REST once the stream has ended81 for t in tabs { el(app, "inspector.tab.\(t)").click(); sleep(4); shot(app, "study-finished-\(t)") }82 if el(app, "run.gate").exists { shot(app, "study-gate") }83 el(app, "usage.line").click(); sleep(1); shot(app, "study-usage-popover"); app.typeKey(.escape, modifierFlags: [])84 if el(app, "provenance.button").exists { el(app, "provenance.button").click(); sleep(1); shot(app, "study-provenance"); app.typeKey(.escape, modifierFlags: []) }85 if el(app, "answer.trace").exists { el(app, "answer.trace").click(); _ = el(app, "trace.done").waitForExistence(timeout: 15); sleep(2); shot(app, "study-trace"); el(app, "trace.done").click() }86 // the study is now in the Studies scope87 el(app, "scope.studies").click(); sleep(4); shot(app, "study-in-studies-list")88 }89}90