SPB Git

Repository Public

ResearchNextTests.swift 10.9 KB 163 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Tests/CoreTests/ResearchNextTests.swift3// Description: Research in 2.0 — unit tests of the v6 chat request and of the frame tap the app feeds the shared kit with; integration tests of the research routes against the REAL VQuant Next test backend (enabled by VQ_NEXT_URL + VQ_NEXT_TOKEN; no model call).4import Foundation5import Testing6@testable import VQuantCore78@Suite struct ResearchNextUnitTests {9    @Test func chatRequestV6CarriesResearchModeAutoBudgetAndProject() throws {10        let r = ChatRequest(query: "q", sessionId: "s-1", history: [], mode: .research, model: "auto", budget: ChatBudget(maxCostUsd: 3, maxMinutes: 500), projectId: "proj_1")11        let b = r.body12        #expect(b["mode"] == "research" && b["model"] == "auto" && b["sessionId"] == "s-1" && b["projectId"] == "proj_1")13        #expect(b["budget"]?["maxCostUsd"]?.doubleValue == 3)14        #expect(b["budget"]?["maxMinutes"]?.doubleValue == 240)   // the server's ceiling: a larger value would be refused whole15    }1617    @Test func chatRequestWithoutBudgetSendsNoneAndRefusesAnInvalidProjectId() {18        let b = ChatRequest(query: "q", sessionId: nil, history: [], mode: .fast, model: "claude-sonnet-5-5", budget: ChatBudget(), projectId: "not a valid id!").body19        #expect(b["budget"] == nil && b["projectId"] == nil && b["sessionId"] == nil)20        let zero = ChatRequest(query: "q", sessionId: nil, history: [], mode: .fast, model: "m", budget: ChatBudget(maxCostUsd: 0, maxMinutes: nil)).body21        #expect(zero["budget"] == nil)22    }2324    @Test func eventsOfBackend6AreKnownAndTextAfterTheQualityGateIsTheSameAnswer() throws {25        var s = RunState()26        for line in [#"{"type":"research_workspace","workspaceId":"s-1","kind":"session"}"#, #"{"type":"model_routed","model":"claude-haiku-5-5"}"#, #"{"type":"text","content":"Persistence 0.97."}"#,27                     #"{"type":"quality_gate","passed":false,"findings":[]}"#, #"{"type":"text","content":" Limitation: n = 400."}"#, #"{"type":"provenance","sources":[]}"#,28                     #"{"type":"approval_request","approval":{"id":"a"}}"#, #"{"type":"brand_new"}"#] {29            s.apply(GenerationEvent(try JSONValue.parse(Data(line.utf8))))30        }31        #expect(s.answer == "Persistence 0.97. Limitation: n = 400.")32        #expect(s.unknownEvents == ["brand_new": 1])33    }3435    @Test func everyFrameReachesTheTapOnceAndInOrderAcrossAReconnection() async {36        let t = MockTransport(); let (api, _) = makeAPI(t)37        let opening = #"data: {"type":"job","id":"job-1","sessionId":"s-1","query":"q"}"#38        let run = [#"{"type":"research_workspace","workspaceId":"s-s-1","kind":"session"}"#, #"{"type":"code_execution","agent":"lead","ok":true,"experimentId":"e1"}"#,39                   #"{"type":"experiment","experiment":{"id":"x1","status":"failed"}}"#, #"{"type":"text","content":"a"}"#, #"{"type":"quality_gate","passed":true,"findings":[]}"#,40                   #"{"type":"done","sessionId":"s-1","usage":{"inputTokens":10,"outputTokens":2,"costUsd":null,"knownCostUsd":0}}"#]41        t.on("POST", "/api/chat", .sse([opening] + events(Array(run.prefix(3))), cut: true))42        t.on("GET", "/api/generations/job-1/stream", .sse([opening] + events(from: 1, Array(run.dropFirst(1)))))43        var runner = GenerationRunner(api: api); runner.backoff = { _ in .milliseconds(1) }44        let seen = Seen()45        runner.onFrame = { f in seen.add(f) }46        var last: GenerationRunner.Update?47        for await u in runner.start(ChatRequest(query: "q", sessionId: nil, history: [], mode: .research, model: "auto")) { last = u }48        #expect(last?.phase == .finished)49        let frames = seen.all50        // numbered frames: each index exactly once, ascending (the replayed 1 and 2 are not repeated)51        #expect(frames.compactMap(\.id) == [0, 1, 2, 3, 4, 5])52        #expect(frames.last?.json["usage"]?["costUsd"] == .null)53    }54}5556private final class Seen: @unchecked Sendable {57    private let lock = NSLock(); private var frames: [SSEFrame] = []58    func add(_ f: SSEFrame) { lock.withLock { frames.append(f) } }59    var all: [SSEFrame] { lock.withLock { frames } }60}6162// MARK: - against the real test backend (port 5058)6364private enum Next {65    static let url = ProcessInfo.processInfo.environment["VQ_NEXT_URL"].flatMap(URL.init(string:))66    static let token = ProcessInfo.processInfo.environment["VQ_NEXT_TOKEN"]67    static var enabled: Bool { url?.port == 5058 && ["localhost", "127.0.0.1"].contains(url?.host ?? "") && token?.hasPrefix("vqn_") == true }68    static func client() -> APIClient {69        APIClient(baseURL: url!, identity: ClientIdentity(version: "2.0.0", build: "1", deviceName: "mac-test integration"), tokens: MemoryTokenStore(token), transport: URLSessionTransport())70    }71}7273@Suite(.serialized, .enabled(if: Next.enabled, "set VQ_NEXT_URL=http://127.0.0.1:5058 and VQ_NEXT_TOKEN (eval \"$(Scripts/dev-session.sh)\")"))74struct ResearchNextIntegrationTests {75    @Test func modelsCatalogueAndRoutePreview() async throws {76        let c = Next.client()77        let cat = try await c.json(.get, "/api/v2/models")78        #expect(cat["default"]?.stringValue?.isEmpty == false && cat["auto"]?["id"] == "auto")79        let providers = cat["providers"]?.arrayValue ?? []80        #expect(providers.contains { $0["id"] == "anthropic" })81        let selectable = providers.flatMap { $0["models"]?.arrayValue ?? [] }.filter { $0["selectable"] == true }82        #expect(!selectable.isEmpty)83        for m in selectable { #expect(m["id"]?.stringValue != nil && m["provider"]?.stringValue != nil && m["available"] == true) }84        let route = try await c.json(.get, "/api/v2/models/route", query: ["mode": "fast", "q": "hello"])85        #expect(route["requested"] == "auto" && route["reason"]?.stringValue?.isEmpty == false)86        #expect(selectable.contains { $0["id"] == route["model"] })87    }8889    @Test func sandboxStatesEnforcementLimitsAndUploadCeiling() async throws {90        let j = try await Next.client().json(.get, "/api/v2/research/sandbox")91        #expect(j["sandbox"]?["enforced"]?.boolValue != nil)92        #expect((j["limits"]?.objectValue ?? [:]).isEmpty == false && (j["uploadMaxBytes"]?.intValue ?? 0) > 0)93        #expect(j["network"]?["default"]?.arrayValue != nil)94    }9596    @Test func createWorkspaceUploadCSVInspectThenListAndDetail() async throws {97        let c = Next.client()98        let created = try await c.json(.post, "/api/v2/research/sessions", body: ["title": .string("mac-test-integration \(Int(Date().timeIntervalSince1970))")])99        let sid = try #require(created["sessionId"]?.stringValue)100        let ws = try #require(created["session"]?["id"]?.stringValue)101        #expect(ws == "s-\(sid)" && created["session"]?["kind"] == "session")102        let base = "/api/v2/research/sessions/" + APIClient.segment("session:\(sid)")103104        // multipart upload (field `file`)105        func upload(_ name: String, _ bytes: Data) async throws -> (Int, JSONValue?) {106            var req = try c.request(.post, base + "/upload")107            let boundary = "vq-test-\(UUID().uuidString)"108            var body = Data("--\(boundary)\r\nContent-Disposition: form-data; name=\"file\"; filename=\"\(name)\"\r\nContent-Type: application/octet-stream\r\n\r\n".utf8)109            body.append(bytes); body.append(Data("\r\n--\(boundary)--\r\n".utf8))110            req.httpBody = body; req.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")111            let (d, http) = try await URLSessionTransport().data(for: req)112            return (http.statusCode, try? JSONValue.parse(d))113        }114        let csv = "date,close\n2026-01-02,100.5\n2026-01-05,101.25\n2026-01-06,\n2026-01-06,\n"115        let (status, up) = try await upload("mac-test-tiny.csv", Data(csv.utf8))116        #expect(status == 201 && up?["file"]?["path"] == "data/mac-test-tiny.csv" && up?["file"]?["bytes"]?.intValue == csv.utf8.count)117        #expect(up?["file"]?["sha256"]?.stringValue?.count == 64)118        // a file whose bytes do not match its extension is refused with a sentence (415)119        let (bad, why) = try await upload("mac-test-fake.parquet", Data("not parquet".utf8))120        #expect(bad == 415 && why?["error"]?.stringValue?.isEmpty == false)121122        let ins = try await c.json(.get, base + "/inspect", query: ["path": "data/mac-test-tiny.csv"])123        let d = try #require(ins["dataset"])124        #expect(d["format"] == "csv" && d["rows"]?.intValue == 4)125        let cols = d["columns"]?.arrayValue ?? []126        #expect(cols.map { $0["name"]?.stringValue } == ["date", "close"])127        let missing = cols.last?["missing"]?.intValue, dup = d["duplicatedRows"]?.intValue128        #expect(missing == 2)129        #expect(dup == 1)130        let timeCols = (d["timeColumns"]?.arrayValue ?? []).compactMap(\.stringValue), head = d["head"]?.arrayValue?.count ?? 0131        #expect(timeCols.contains("date"))132        #expect(head > 0)133134        let detail = try await c.json(.get, base)135        #expect(detail["session"]?["id"]?.stringValue == ws)136        for key in ["files", "experiments", "executions", "runs", "notes", "library", "approvals"] { #expect(detail[key]?.arrayValue != nil, "detail.\(key) is a list") }137        #expect(detail["environment"]?.objectValue != nil)138        let detailFiles = (detail["files"]?.arrayValue ?? []).compactMap { $0["path"]?.stringValue }139        #expect(detailFiles.contains("data/mac-test-tiny.csv"))140        let files = try await c.json(.get, base + "/files", query: ["path": "", "recursive": "1"])141        let listed = (files["files"]?.arrayValue ?? []).compactMap { $0["path"]?.stringValue }142        #expect(listed.contains("data/mac-test-tiny.csv"))143        let text = try await c.json(.get, base + "/file", query: ["path": "data/mac-test-tiny.csv", "offset": "2", "limit": "2"])144        let from = text["from"]?.intValue, to = text["to"]?.intValue, total = text["totalLines"]?.intValue, body = text["text"]?.stringValue ?? ""145        #expect(from == 2)146        #expect(to == 3)147        #expect((total ?? 0) >= 5)   // the server counts the empty line after the last newline148        #expect(body.hasPrefix("2026-01-02"))149150        let list = try await c.json(.get, "/api/v2/research/sessions")151        let mine = try #require(list["sessions"]?.arrayValue?.first { $0["id"]?.stringValue == ws })152        let ref = mine["ref"]?.stringValue, title = mine["title"]?.stringValue ?? ""153        #expect(ref == sid)154        #expect(title.hasPrefix("mac-test-integration"))155        #expect(mine["sessionIds"]?.arrayValue != nil)156        #expect(mine["updatedAt"]?.doubleValue != nil)157        // a conversation without a workspace is a 404, not an empty study158        await #expect { try await c.json(.get, "/api/v2/research/sessions/" + APIClient.segment("session:mac-test-none-00000000")) } throws: { ($0 as? APIError)?.kind == .notFound }159        let approvals = try await c.json(.get, "/api/v2/research/approvals", query: ["status": "pending"])160        #expect(approvals["approvals"]?.arrayValue != nil)161    }162}163
Files main
Menu

Theme