SPB Git

Repository Public

ResearchAgentTests.swift 30.9 KB 355 lines swift

1import Foundation2import Testing3@testable import VQuantKit4import VQuantKitTesting56private enum R {7    static func json(_ name: String) throws -> VQJSON { try Fixture.json("research/\(name)") }8    /// The generation stream of the real study (recorded by the web team: client/src/lib/research/__fixtures__/stream-study.json).9    static func stream() throws -> [VQJSON] { try json("stream_study.json").arrayValue ?? [] }10    static let generation = "0bc4b1ce-439f-451e-a893-f49987b53db6"11    static let workspace = "s-3d04acf2-235e-46e3-8c2d-bf26b0ba3c81"12}1314/// The real research session recorded on the 5059 copy: plan, code, installs, 10 parallel runs, 4 failed executions15/// of 56, 9 experiments (one failed, one abandoned), 14 figures, quality gate.16@Suite struct ResearchReducerTests {17    @Test func theStreamReproducesTheStudy() throws {18        let s = VQResearchReducer.replay(try R.stream())19        #expect(s.workspaceId == R.workspace && s.workspaceKind == "session" && s.generationId == R.generation && s.finished)20        #expect(s.objective?.hasPrefix("J ai téléversé data/us_macro_quarterly.csv") == true)21        #expect(s.plan.count == 7 && s.plan[0].status == .inProgress && s.plan[1].status == .pending)22        let c = s.counts23        #expect(c.executions == 56 && c.failed == 4)                    // 46 notebook cells + 10 script runs; the failed ones are kept24        #expect(c.cells == 46 && c.runs == 10 && c.running == 0 && c.runsRunning == 0)25        #expect(c.experiments == 9 && c.figures == 14 && c.installs == 1 && c.pendingApprovals == 0)26        #expect(s.experiments.map(\.status.rawValue).filter { $0 != "ok" } == ["failed", "abandoned"])27        #expect(s.runs.allSatisfy { $0.status == .done && $0.progress == 1 && $0.finishedAt != nil && $0.code?.hasPrefix("code/") == true })28        let failed = s.executions.filter { $0.status == .failed }29        #expect(failed.count == 4 && failed.allSatisfy { ($0.error?.isEmpty == false) && $0.code != nil })30        #expect(failed[0].error?.contains("ModuleNotFoundError") == true && failed[0].title == "Installation de ruptures et module commun")31        #expect(s.executions.allSatisfy { $0.ledgerId != nil && $0.codeHash != nil && $0.codePath?.hasPrefix("logs/cells/") == true && $0.durationMs != nil })32        #expect(s.installs == [.init(key: s.installs[0].key, spec: "ruptures", ok: true)])33        #expect(s.datasets == [.init(path: "data/us_macro_quarterly.csv", rows: 203, columns: 13, bytes: 17424)])34        #expect(s.files.count == 1 && s.files[0].action == .library && s.notes.count == 1 && s.notes[0].kind == .finding)35        // figures: every one linted, with its three exports; three keep a defect the linter could not repair36        #expect(s.figures.allSatisfy { $0.lintPassed != nil && $0.files?.png != nil && $0.files?.svg != nil && $0.files?.pdf != nil && $0.executionKey != nil })37        #expect(s.figures.filter { $0.lintPassed == false }.count == 3)38        #expect(Set(s.figures.compactMap(\.executionKey)).isSubset(of: Set(s.executions.map(\.key))))39        // quality gate40        #expect(s.gates.count == 1 && s.gates[0].passed == false && s.gates[0].findings.map(\.id).contains("figures"))41        #expect(s.gateFindings.first?.isOpen == true && s.gateFindings.last?.isOpen == false)42        // the timeline is the feed: one entry per entity, in order of first appearance43        #expect(s.timeline.first?.kind == .workspace && s.timeline.contains { $0.kind == .gate })44        #expect(Set(s.timeline.map(\.key)).count == s.timeline.count)45        let kinds = Dictionary(grouping: s.timeline, by: \.kind).mapValues(\.count)46        #expect(kinds[.execution] == 46 && kinds[.run] == 10 && kinds[.experiment] == 9 && kinds[.figure] == 14 && kinds[.install] == 1 && kinds[.plan] == 1 && kinds[.dataset] == 1)47    }4849    @Test func replayFromAnIndexAndDuplicatesChangeNothing() throws {50        let events = try R.stream()51        let once = VQResearchReducer.replay(events, firstIndex: 0)52        #expect(once.lastIndex == events.count - 1)53        // the whole stream delivered again from index 0 (a re-attach that restarts): ignored by index54        #expect(VQResearchReducer.replay(events, onto: once, firstIndex: 0) == once)55        // a re-attach from the middle with an overlap56        let half = VQResearchReducer.replay(Array(events[..<150]), firstIndex: 0)57        #expect(VQResearchReducer.replay(Array(events[120...]), onto: half, firstIndex: 120) == once)58        // without indices (a transport that does not know them): entities with an id are upserts, so the entity59        // events delivered twice leave the counts unchanged60        var s = VQResearchReducer.replay(events)61        let before = s.counts62        for e in events where ["research_run", "experiment", "code_execution", "figure_lint", "research_note", "dataset", "research_workspace", "quality_gate", "tool_start", "plan"].contains(e.str("type")) {63            VQResearchReducer.reduce(&s, e)64        }65        #expect(s.counts == before && s.plan.count == 7)66        // a late duplicate of a run's start does not bring it back to life67        let start = try #require(events.first { $0.str("type") == "research_run" && $0["run"]?.str("status") == "running" })68        VQResearchReducer.reduce(&s, start)69        #expect(s.counts.runsRunning == 0)70    }7172    @Test func unknownAndMalformedEventsAreIgnored() throws {73        var s = VQResearchReducer.replay(try R.stream())74        let before = s75        for e: VQJSON in [["type": "research_telepathy", "x": 1], ["type": "experiment"], ["type": "research_run", "run": "nope"], ["type": "figure_lint", "issues": "x", "files": ["png": "artifacts/figures/new.png"]],76                          ["no": "type"], "scalar", ["type": "code_execution", "experimentId": .string(before.executions[0].ledgerId!)]] {77            VQResearchReducer.reduce(&s, e)78        }79        #expect(s.figures.count == before.figures.count + 1)            // the tolerant figure_lint was still a figure80        s.figures.removeLast(); s.timeline.removeLast(); s.counter = before.counter81        if let k = s.executions.firstIndex(where: { $0.figureKeys != before.execution($0.key)?.figureKeys }) { s.executions[k].figureKeys.removeLast() }82        #expect(s == before)83        guard case .other(let type, _) = VQResearchEvent(json: ["type": "research_telepathy"]) else { Issue.record("expected other"); return }84        #expect(type == "research_telepathy" && VQResearchEvent.types.count == 12)85    }8687    @Test func typedEventsOfTheRecordedStream() throws {88        var seen = Set<String>()89        for e in try R.stream() where VQResearchEvent.types.contains(e.str("type")) {90            seen.insert(e.str("type"))91            switch VQResearchEvent(json: e) {92            case .workspace(let id, let kind): #expect(id == R.workspace && kind == "session")93            case .codeExecution(let agent, _, let hash, let path, let lines, _, let ms, _, let id): #expect(agent == "lead" && hash != nil && path != nil && (lines ?? 0) > 0 && ms != nil && id != nil)94            case .run(let run): #expect(!run.id.isEmpty && run.status.known != nil && run.network == .default)95            case .experiment(let x): #expect(x.kind == .experiment && x.seq != nil && x.status.known != nil)96            case .figureLint(let title, _, _, let files): #expect(title != nil && files?.png != nil)97            case .qualityGate(let passed, let findings): #expect(!passed && findings.count >= 4)98            case .packageInstall(let spec, let ok): #expect(spec == "ruptures" && ok)99            case .dataset(let d): #expect(d.rows == 203)100            case .file(let f): #expect(f.action == .library && f.path == "vqlib/fc_eval.py")101            case .note(let n): #expect(n.kind == .finding)102            case .approvalRequest, .approvalResolved: break103            case .other: Issue.record("a known research event decoded as other: \(e.str("type"))")104            }105        }106        #expect(seen == ["research_workspace", "dataset", "research_file", "code_execution", "package_install", "research_run", "experiment", "research_note", "figure_lint", "quality_gate"])107    }108109    /// No approval occurred in the recorded study: these events are written from CONTRACT §9.6–9.7.110    @Test func approvalsLifecycleAndCountdown() {111        var s = VQResearchState()112        let created = 1_795_300_000_000.0113        let request: VQJSON = ["type": "approval_request", "approval": ["id": "ap1", "action": "open_network", "reason": "Télécharger la série FRED CPIAUCSL", "detail": ["host": "fred.stlouisfed.org", "script": "code/fetch.py"], "status": "pending", "createdAt": .number(created), "resolvedAt": .null]]114        VQResearchReducer.reduce(&s, request)115        let a = s.pendingApprovals[0]116        #expect(a.action == .openNetwork && a.deadline == Date(timeIntervalSince1970: created / 1000 + 900))117        #expect(a.details(.fr).map(\.label) == ["Destination", "Script"] && a.title(.en) == "Open the network for one run" && !a.scope(.fr).isEmpty)118        let start = Date(timeIntervalSince1970: created / 1000)119        #expect(a.countdown(now: start).label == "15:00" && a.countdown(now: start.addingTimeInterval(13 * 60 + 30)).urgent && a.countdown(now: start.addingTimeInterval(13 * 60 + 30)).label == "01:30")120        #expect(a.isAnswerable(now: start.addingTimeInterval(899)) && !a.isAnswerable(now: start.addingTimeInterval(900)))121        #expect(a.countdown(now: start.addingTimeInterval(2000)).expired && a.countdown(now: start.addingTimeInterval(450)).elapsedShare == 0.5)122        VQResearchReducer.reduce(&s, ["type": "approval_resolved", "approval": ["id": "ap1", "status": "approved", "resolvedAt": .number(created + 60_000)]])123        #expect(s.pendingApprovals.isEmpty && s.approvals[0].status == .approved && s.approvals[0].reason == "Télécharger la série FRED CPIAUCSL")124        VQResearchReducer.reduce(&s, request)                              // replayed after its resolution: not reopened125        #expect(s.approvals.count == 1 && s.approvals[0].status == .approved)126        let budget = VQResearchApproval(json: ["id": "b", "action": "raise_budget", "detail": ["maxCostUsd": 5, "maxMinutes": 90], "status": "pending"])127        #expect(budget.details(.en, locale: .enCA).map(\.value) == ["US$5.00", "90 min"] && budget.countdown(now: start).remaining == nil)128        #expect(VQResearchApproval(json: ["id": "z", "action": "launch_rocket"]).action.known == nil)129    }130131    @Test func liveCellsOutputsAndInterruption() {132        var s = VQResearchState()133        let nb = VQResearchReducer.notebookTool134        VQResearchReducer.reduce(&s, ["type": "tool_start", "tool": .string(nb), "toolId": "t1", "input": ["code": "print(1)\nprint(2)", "description": "Deux lignes"]])135        VQResearchReducer.reduce(&s, ["type": "tool_start", "tool": .string(nb), "toolId": "t2", "input": ["code": "x = 1", "description": "Autre"]])136        #expect(s.counts.running == 2 && s.executions[0].lines == 2 && s.executions[0].code == "print(1)\nprint(2)")137        VQResearchReducer.reduce(&s, ["type": "code_execution", "agent": "lead", "description": "Autre", "codeHash": "abc", "code": "logs/cells/abc.py", "lines": 1, "ok": true, "durationMs": 12, "experimentId": "e2"])138        #expect(s.executions[1].status == .ok && s.executions[0].status == .running)          // paired by description, not by order139        VQResearchReducer.reduce(&s, ["type": "tool_complete", "tool": .string(nb), "toolId": "t2"])140        VQResearchReducer.reduce(&s, ["type": "tool_result", "tool": .string(nb), "result": ["output": "42\n[Notebook session — variables in memory: x]"]])141        #expect(s.executions[1].output == "42")142        VQResearchReducer.reduce(&s, ["type": "text", "content": "Voici "]); VQResearchReducer.reduce(&s, ["type": "text", "content": "le rapport."])143        #expect(s.report == "Voici le rapport.")144        VQResearchReducer.reduce(&s, ["type": "tool_error", "tool": .string(nb), "toolId": "t1", "error": "Execution stopped after 600 s (time limit)."])145        #expect(s.executions[0].status == .timeout)146        VQResearchReducer.reduce(&s, ["type": "tool_start", "tool": .string(nb), "toolId": "t3", "input": ["code": "while True: pass"]])147        VQResearchReducer.reduce(&s, ["type": "budget", "kind": "cost", "limit": 2, "spent": 2.1])148        VQResearchReducer.reduce(&s, ["type": "done"])149        #expect(s.finished && s.executions[2].status == .interrupted && s.budgetHits.count == 1 && s.counts.failed == 2)150        VQResearchReducer.reduce(&s, ["type": "plan", "steps": [["title": "A", "status": "done"], ["title": "B", "status": "in_progress"]]])151        VQResearchReducer.reduce(&s, ["type": "plan", "steps": [["title": "A", "status": "done"], ["title": "B", "status": "done"]]])152        #expect(s.planRevisions == 0)153        VQResearchReducer.reduce(&s, ["type": "plan", "steps": [["title": "A", "status": "done"], ["title": "C", "status": "pending"]]])154        #expect(s.planRevisions == 1 && s.timeline.filter { $0.kind == .plan }.count == 2)   // a re-plan is a decision in the timeline155    }156}157158@Suite struct ResearchRESTTests {159    @Test func sessionDetailAndLedger() throws {160        let d = VQResearchSessionDetail(json: try R.json("session.json"))161        #expect(d.session.id == R.workspace && d.session.kind == .session && d.session.sessionIds.count == 1 && d.session.title == "US macro: inflation, unemployment, rates")162        #expect(d.files.count == 230 && d.experiments.count == 10 && d.executions.count == 72 && d.runs.count == 11 && d.notes.count == 2 && d.library.count == 1 && d.approvals.isEmpty)163        #expect(d.environment.packages == 89 && d.environment.overlay.contains("ruptures==1.1.10") && d.environment.lockfile == "environment.lock.txt" && d.environment.hash != nil)164        #expect(d.library[0].exports.contains("dm_test"))165        let failed = d.executions.filter { $0.status == .failed }166        #expect(failed.count == 5 && failed.allSatisfy { $0.kind == .execution && $0.error != nil && $0.status.known?.isFailure == true })167        let abandoned = try #require(d.experiments.first { $0.status == .abandoned })168        #expect(abandoned.notes?.contains("BIC decreased monotonically") == true && abandoned.kind == .experiment)169        #expect(d.experiments[0].metrics?["infl ADF p full"] == .number(0.027) && d.experiments[0].artifacts == ["artifacts/stationarity.csv"])170        #expect(d.runs[0].code == "code/gr_run.py" && d.runs[0].duration != nil && d.runs.last?.args == ["expanding", "0"])171        let ledger = try R.json("experiments.json").models("experiments", as: VQLedgerEntry.self)172        #expect(ledger.count == 10 && zip(ledger, ledger.dropFirst()).allSatisfy { ($0.seq ?? 0) < ($1.seq ?? 0) })      // oldest first173        #expect(try R.json("executions.json").models("experiments", as: VQLedgerEntry.self).count == 72)174        let sessions = try R.json("sessions.json").models("sessions", as: VQResearchSession.self)175        #expect(sessions.count == 3 && sessions[0].id == R.workspace)176        #expect(VQResearchSession.reference(sessionId: "abc") == "session:abc")177    }178179    @Test func filesTreePreviewsAndTables() throws {180        let files = try R.json("files.json").models("files", as: VQResearchFile.self)181        let tree = VQResearchFileTree.build(files)182        #expect(tree.map(\.name) == ["artifacts", "code", "data", "logs", "runs", "environment.lock.txt"])            // directories first183        #expect(tree.reduce(0) { $0 + $1.fileCount } == files.filter { !$0.isDirectory }.count)184        #expect(tree[0].childNodes != nil && tree[5].childNodes == nil && tree[0].bytes > 1_000_000)185        #expect(VQResearchFileTree.kind(of: "code/fc_ucsv.py") == .code && VQResearchFileTree.kind(of: "a/b.PNG") == .image && VQResearchFileTree.kind(of: "environment.lock.txt") == .text)186        #expect(VQResearchFileTree.isTextual("logs/executions.jsonl") && !VQResearchFileTree.isTextual("x.parquet") && VQResearchFileTree.isDataset("data/us_macro_quarterly.csv"))187188        let code = VQResearchFilePreview(json: try R.json("file_code.json"))189        #expect(code.path == "code/fc_ucsv.py" && code.totalLines == 29 && code.from == 1 && code.to == 29 && !code.truncated && code.text.contains("import"))190        let page = VQResearchFilePreview(json: try R.json("file_jsonl.json"))191        #expect(page.truncated && page.to == 5 && page.totalLines == 73)192        let csv = VQDelimitedText.parse(VQResearchFilePreview(json: try R.json("file_csv.json")).text)193        #expect(csv.delimiter == "," && csv.header.count > 3 && csv.rows.count == 21 && csv.rows.allSatisfy { $0.count == csv.header.count } && !csv.truncated)194        let tricky = VQDelimitedText.parse("a;b\n\"x;1\";\"say \"\"hi\"\"\"\n\"line\nbreak\";2\r\n3;4\n", maxRows: 2)195        #expect(tricky.delimiter == ";" && tricky.rows == [["x;1", "say \"hi\""], ["line\nbreak", "2"]] && tricky.truncated)196        #expect(VQDelimitedText.parse("").header.isEmpty)197198        let figures = VQResearchState.figureFiles(in: files.map(\.path))199        #expect(figures.count == 21 && figures.allSatisfy { $0.files.png != nil } && figures.filter { $0.files.svg != nil && $0.files.pdf != nil }.count == 20)200    }201202    @Test func inspectionRunLogsAndSandbox() throws {203        let i = VQDatasetInspection(json: try R.json("inspect.json"))204        #expect(i.path == "data/us_macro_quarterly.csv" && i.format == "csv" && i.rows == 203 && i.columnCount == 13 && i.columns.count == 13 && i.timeColumns == ["period"] && i.duplicatedRows == 0)205        #expect(i.columns[0].parsesAsDate && i.columns[0].range.count == 2 && i.columns[1].summary["mean"] != nil && i.columns[1].missingShare == 0 && i.head.count == 8 && i.head[0]["period"] == "1959Q1")206        let logs = VQRunLogs(json: try R.json("run_detail.json"))207        #expect(logs.run.status == .done && logs.run.exitCode == 0 && logs.stdout.contains("linear_expanding") && logs.stderr.isEmpty)208        let sb = VQSandboxStatus(json: try R.json("sandbox.json"))209        #expect(sb.enforced && sb.mechanism == "sandbox-exec" && sb.limits["memoryMb"] == 8192 && sb.defaultNetwork.contains("pypi.org") && sb.uploadMaxBytes == 209_715_200 && sb.openNetwork != nil)210        #expect(!VQSandboxStatus(json: ["sandbox": ["enforced": false, "mechanism": .null, "note": "No isolation on this host."]]).enforced)211        #expect(try R.json("approvals_pending.json").models("approvals", as: VQResearchApproval.self).isEmpty)212        #expect(VQAnalysisMode.modes(ofConfig: ["modes": ["fast", "standard", "deep", "ultra", "research", "hyper"]]).map(\.known) == [.fast, .standard, .deep, .ultra, .research, nil])213        #expect(VQAnalysisMode.allCases.contains(.research) && VQAnalysisMode.research.isAutonomousResearch)214    }215216    @Test func storedConversationCarriesTheResearchSummary() throws {217        let conversation = try R.json("conversation.json")218        let research = VQMessageResearch.ofSession(conversation)219        #expect(research.count == 2)220        let first = try #require(research[0])221        #expect(first.workspaceId == R.workspace && first.executionsTotal == 56 && first.executionsFailed == 4 && first.experiments.count == 9)222        #expect(first.qualityGate?.count == 4 || (first.qualityGate?.count ?? 0) >= 4)223        #expect(research[1]?.executionsTotal == 16 && research[1]?.executionsFailed == 1)224        #expect(VQMessageExtras.ofSession(conversation)[0].generationId == R.generation)225    }226227    /// The finished study rebuilt from REST + the stored conversation gives the same numbers as its stream did.228    @Test func snapshotMatchesTheStream() throws {229        let detail = VQResearchSessionDetail(json: try R.json("session.json"))230        let whole = VQResearchState(detail: detail, conversation: try R.json("conversation.json"))231        #expect(whole.finished && whole.workspaceId == R.workspace && whole.objective?.hasPrefix("J ai téléversé") == true)232        #expect(whole.counts.executions == 72 && whole.counts.failed == 5 && whole.counts.runs == 11 && whole.counts.experiments == 10 && whole.figures.count == 21)   // both generations233        #expect(whole.gates.count == 2 && whole.report.contains("Giacomini") && whole.datasets.map(\.path) == ["data/us_macro_quarterly.csv"] && whole.notes.count == 2)234        #expect(whole.timeline.first?.kind == .workspace && zip(whole.timeline.dropFirst(), whole.timeline.dropFirst(2)).allSatisfy { a, b in (a.at ?? .distantFuture) <= (b.at ?? .distantFuture) })235236        let study = whole.filtered(generationId: R.generation)237        let live = VQResearchReducer.replay(try R.stream())238        #expect(study.counts.executions == live.counts.executions && study.counts.failed == live.counts.failed)          // 56 and 4239        #expect(study.counts.cells == 46 && study.counts.runs == 10 && study.counts.experiments == 9 && study.figures.count == 14)240        #expect(Set(study.executions.compactMap(\.ledgerId)) == Set(live.executions.compactMap(\.ledgerId)))241        #expect(Set(study.runs.map(\.id)) == Set(live.runs.map(\.id)) && study.experiments.map(\.id) == live.experiments.map(\.id))242        #expect(Set(study.figures.compactMap { $0.files?.png }) == Set(live.figures.compactMap { $0.files?.png }))243        #expect(study.gates.first?.findings.map(\.id) == live.gates.first?.findings.map(\.id))244        // what only REST has: console outputs of the successful cells, the agent's notes on the abandoned experiment245        let withOutput = study.executions.filter { $0.status == .ok && $0.output != nil }.count246        #expect(withOutput == 32, "\(withOutput) outputs")      // the other ten cells printed nothing (figures only)247        #expect(study.executions.filter { $0.status == .failed }.allSatisfy { $0.error != nil })248        #expect(study.experiments.first { $0.status == .abandoned }?.notes != nil)249        // what the server does not keep: the linter's verdicts250        #expect(study.figures.allSatisfy { $0.lintPassed == nil })251252        // a second device joining mid-study: live events over the snapshot of the earlier generation253        let second = "c776e64f-f227-42c2-ae53-ca7ed1a1becc"254        var joining = VQResearchState()255        VQResearchReducer.reduce(&joining, ["type": "job", "id": .string(second), "sessionId": "3d04acf2-235e-46e3-8c2d-bf26b0ba3c81", "query": "Suite"])256        VQResearchReducer.reduce(&joining, ["type": "research_run", "run": ["id": "new-run", "label": "GR", "code": "code/gr_run.py", "status": "running"]])257        let merged = joining.merged(over: whole.filtered(generationId: R.generation))258        #expect(merged.runs.count == 11 && merged.counts.runsRunning == 1 && merged.executions.count == 46 && merged.objective == "Suite")259    }260}261262@Suite @MainActor struct ResearchStoreTests {263    @Test func storeFollowsTheStreamAndAnswersApprovals() async throws {264        let store = VQResearchSessionStore()265        var changed = 0266        for (i, e) in try R.stream().enumerated() { if store.apply(e, index: i) { changed += 1 } }267        #expect(store.counts.executions == 56 && store.counts.failed == 4 && store.experiments.count == 9 && store.figures.count == 14 && store.runs.count == 10)268        #expect(store.plan.count == 7 && store.installs.count == 1 && store.gateFindings.count >= 4 && store.pendingApprovals.isEmpty && !store.isRunning)269        #expect(store.objective != nil && store.timeline.count > 80 && store.workspaceId == R.workspace)270        #expect(changed < (try R.stream().count))                        // tool_complete, sources, followups… publish nothing271        #expect(!store.apply(try R.stream()[10], index: 10))             // a replay272273        // REST rebuild, then a live approval answered through the client274        let transport = VQScriptedTransport([275            .init(json: try R.json("session.json")), .init(json: ["approvals": []]), .init(json: try R.json("sandbox.json")),276            .init(json: ["approval": ["id": "ap1", "action": "open_network", "status": "approved", "resolvedAt": 1_795_300_060_000]]),277            .init(status: 404, json: ["error": "Not found."]),278        ])279        let rest = VQResearchSessionStore(client: VQResearchAgentClient(client: VQClient(transport: transport)))280        await rest.reload(sessionId: "3d04acf2-235e-46e3-8c2d-bf26b0ba3c81", conversation: try R.json("conversation.json"))281        #expect(transport.requests[0].path == "/api/v2/research/sessions/session%3A3d04acf2-235e-46e3-8c2d-bf26b0ba3c81")282        #expect(rest.counts.executions == 72 && rest.detail?.files.count == 230 && rest.sandbox?.enforced == true && rest.finalReport.contains("Giacomini") && rest.lastError == nil)283        for id in ["ap1", "ap2"] {284            rest.apply(["type": "approval_request", "approval": ["id": .string(id), "action": "open_network", "reason": "r", "status": "pending", "createdAt": 1_795_300_000_000]])285        }286        #expect(rest.pendingApprovals.count == 2 && rest.counts.executions == 72)           // live events sit on top of the snapshot287        #expect(await rest.answer(rest.pendingApprovals[0], .approve))288        #expect(try VQJSON.parse(transport.requests[3].body!) == ["decision": "approve"] && transport.requests[3].path == "/api/v2/research/approvals/ap1")289        #expect(await rest.answer(rest.pendingApprovals[0], .deny) == false)                // answered elsewhere or expired: 404290        #expect(rest.pendingApprovals.isEmpty && rest.state.approvals.map(\.status.rawValue) == ["approved", "expired"])291    }292}293294@Suite struct ResearchClientTests {295    @Test func requestsOfEveryRoute() async throws {296        let ok = VQScriptedTransport.Reply(json: [:])297        let t = VQScriptedTransport(Array(repeating: ok, count: 12))298        let c = VQResearchAgentClient(client: VQClient(transport: t))299        _ = try await c.sessions(); _ = try await c.createSession(title: "T", projectId: "p1")300        _ = try await c.files("s-1", path: "artifacts"); _ = try await c.textFile("s-1", path: "code/a b.py", offset: 100, limit: 50)301        _ = try await c.fileData("s-1", path: "artifacts/figures/f.png"); _ = try await c.inspect("s-1", path: "data/x.csv")302        _ = try await c.experiments("p-9", kind: .execution); _ = try await c.runs("s-1"); _ = try await c.run("r1", tail: 2000)303        _ = try await c.cancelRun("r1"); _ = try await c.approvals(); _ = try await c.sandbox()304        let r = t.requests305        #expect(r.map { "\($0.method.rawValue) \($0.path)" } == [306            "GET /api/v2/research/sessions", "POST /api/v2/research/sessions", "GET /api/v2/research/sessions/s-1/files", "GET /api/v2/research/sessions/s-1/file",307            "GET /api/v2/research/sessions/s-1/file", "GET /api/v2/research/sessions/s-1/inspect", "GET /api/v2/research/sessions/p-9/experiments", "GET /api/v2/research/sessions/s-1/runs",308            "GET /api/v2/research/runs/r1", "POST /api/v2/research/runs/r1/cancel", "GET /api/v2/research/approvals", "GET /api/v2/research/sandbox"])309        #expect(try VQJSON.parse(r[1].body!) == ["title": "T", "projectId": "p1"])310        #expect(r[2].query == [VQQueryItem("path", "artifacts"), VQQueryItem("recursive", "1")] && r[3].query == [VQQueryItem("path", "code/a b.py"), VQQueryItem("offset", "100"), VQQueryItem("limit", "50")])311        #expect(r[4].query.contains(VQQueryItem("download", "1")) && r[6].query == [VQQueryItem("kind", "execution")] && r[8].query == [VQQueryItem("tail", "2000")] && r[10].query == [VQQueryItem("status", "pending")])312    }313314    @Test func uploadChecksBeforeSendingAndBuildsMultipart() async throws {315        typealias U = VQDatasetUpload316        #expect(U.validate(name: "data", size: 10) == .noExtension && U.validate(name: "a.zip", size: 10) == .unsupported(".zip") && U.validate(name: "a.csv", size: 0) == .empty)317        #expect(U.validate(name: "a.CSV", size: 300, maxBytes: 200) == .tooLarge(bytes: 300, limit: 200) && U.validate(name: "a.csv", size: 10) == nil)318        #expect(U.validate(name: "a.parquet", size: 8, head: Data("PAR1....".utf8)) == nil && U.validate(name: "a.parquet", size: 8, head: Data("a,b\n1,2\n".utf8)) == .contentMismatch(".parquet"))319        #expect(U.validate(name: "a.xlsx", size: 8, head: Data([0x50, 0x4B, 3, 4, 0, 0, 0, 0])) == nil && U.validate(name: "a.csv", size: 8, head: Data([0x50, 0x4B, 3, 4, 0, 0, 0, 0])) == .contentMismatch(".csv"))320        #expect(U.validate(name: "a.csv", size: 6, head: Data("%PDF-1".utf8)) == .contentMismatch(".csv") && U.validate(name: "a.txt", size: 3, head: Data([65, 0, 66])) == .contentMismatch(".txt"))321        #expect(U.validate(name: "a.xls", size: 4, head: Data([0xD0, 0xCF, 0x11, 0xE0])) == nil && U.validate(name: "a.feather", size: 6, head: Data("ARROW1".utf8)) == nil)322        #expect(U.Refusal.tooLarge(bytes: 300 * 1024 * 1024, limit: U.defaultMaxBytes).message(.fr) == "Ce fichier pèse 300,0\u{00A0}Mo ; la limite est de 200,0\u{00A0}Mo.")323        #expect(VQFormat.bytes(1536) == "1,5\u{00A0}ko" && VQFormat.bytes(17424, language: .en) == "17\u{00A0}KB" && VQFormat.bytes(nil) == "—")324325        let t = VQScriptedTransport([.init(status: 201, json: ["file": ["path": "data/prices.csv", "bytes": 12, "sha256": "ab"]]), .init(status: 415, json: ["error": "content does not match"])])326        let c = VQResearchAgentClient(client: VQClient(transport: t))327        let file = try await c.upload("s-1", name: "prices.csv", data: Data("a,b\n1,2\n3,4\n".utf8))328        #expect(file.path == "data/prices.csv" && file.bytes == 12)329        let sent = t.requests[0]330        #expect(sent.path == "/api/v2/research/sessions/s-1/upload" && sent.contentType?.hasPrefix("multipart/form-data; boundary=vq-") == true)331        let body = String(decoding: sent.body!, as: UTF8.self)332        #expect(body.contains("name=\"file\"; filename=\"prices.csv\"") && body.contains("a,b\n1,2\n3,4\n") && body.hasSuffix("--\r\n"))333        await #expect(throws: VQDatasetUpload.Refusal.self) { try await c.upload("s-1", name: "x.exe", data: Data([1])) }      // refused before sending334        #expect(t.requests.count == 1)335        do { _ = try await c.upload("s-1", name: "x.csv", data: Data("a\n".utf8)); Issue.record("415 not thrown") }336        catch let e as VQError { #expect(e.status == 415 && U.message(for: e, .en).contains("does not match")) }337        // the JSON default is untouched for every other request338        #expect(VQRequest(.post, "/x", body: ["a": 1]).contentType == nil)339    }340341    @Test func pythonTokens() {342        let src = "import numpy as np  # maths\n@cache\ndef f(x: int = 3) -> str:\n    s = f\"{x:.2f}\" + 'a\\'b' + r'\\d'\n    return \"\"\"doc\n# not a comment\"\"\" if x >= 1.5e-3 else None\nprint(df.sum())\n"343        let u = Array(src.utf16)344        func text(_ t: VQPythonSyntax.Token) -> String { String(utf16CodeUnits: Array(u[t.range]), count: t.range.count) }345        let tokens = VQPythonSyntax.tokens(src)346        let by = Dictionary(grouping: tokens, by: \.kind).mapValues { $0.map(text) }347        #expect(by[.keyword] == ["import", "as", "def", "return", "if", "else", "None"])348        #expect(by[.comment] == ["# maths"] && by[.decorator] == ["@cache"] && by[.definition] == ["f"])349        #expect(by[.string] == ["\"{x:.2f}\"", "'a\\'b'", "'\\d'", "\"\"\"doc\n# not a comment\"\"\""])350        #expect(by[.number] == ["3", "1.5e-3"] && by[.builtin] == ["int", "str", "print"])                 // df.sum is a method, not the built-in351        #expect(zip(tokens, tokens.dropFirst()).allSatisfy { $0.range.upperBound <= $1.range.lowerBound })352        #expect(VQPythonSyntax.tokens("x = 'unterminated\ny = 2").count == 2 && VQPythonSyntax.tokens("").isEmpty)353    }354}355
Files main
Menu

Theme