Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Sources/VQuantKit/ResearchAgent/VQResearchState.swift3// Description: The view model of a research study and its two builders: the reducer of the generation stream4// (CONTRACT §9.6) and the rebuild from REST + the stored conversation once the stream is gone.5// A port of the web's `client/src/lib/research/{reducer,snapshot}.ts`, so the three platforms show the6// same thing for the same events.7import Foundation89/// The eleven event types §9 adds to a generation stream, decoded tolerantly (an unknown type is `.other`).10/// `VQResearchReducer` consumes raw event JSON directly; this enum is for code that wants one event typed.11public enum VQResearchEvent: Sendable, Equatable {12 case workspace(id: String, kind: String)13 case dataset(VQResearchState.Dataset)14 case file(VQResearchState.FileChange)15 case codeExecution(agent: String, description: String?, codeHash: String?, codePath: String?, lines: Int?, ok: Bool, durationMs: Double?, error: String?, experimentId: String?)16 case packageInstall(spec: String, ok: Bool)17 case run(VQResearchRun)18 case experiment(VQLedgerEntry)19 case note(VQResearchNote)20 case figureLint(title: String?, ok: Bool, issues: [VQResearchState.LintIssue], files: VQResearchState.FigureFiles?)21 case approvalRequest(VQResearchApproval)22 case approvalResolved(VQResearchApproval)23 case qualityGate(passed: Bool, findings: [VQQualityGateFinding])24 case other(type: String, payload: VQJSON)2526 public static let types: Set<String> = ["research_workspace", "dataset", "research_file", "code_execution", "package_install", "research_run", "experiment",27 "research_note", "figure_lint", "approval_request", "approval_resolved", "quality_gate"]2829 public init(json j: VQJSON) {30 let type = j.str("type")31 switch type {32 case "research_workspace": self = .workspace(id: j.str("workspaceId"), kind: j.str("kind", "session"))33 case "dataset": self = .dataset(.init(path: j.str("path"), rows: j.int("rows"), columns: j.int("columns"), bytes: j.int("bytes")))34 case "research_file": self = .file(.init(action: j.open("action", .write), path: j.str("path"), bytes: j.int("bytes"), agent: j.optStr("agent")))35 case "code_execution":36 self = .codeExecution(agent: j.str("agent", "lead"), description: j.optStr("description"), codeHash: j.optStr("codeHash"), codePath: j.optStr("code"), lines: j.int("lines"),37 ok: j.flag("ok"), durationMs: j.num("durationMs"), error: j.optStr("error"), experimentId: j.optStr("experimentId"))38 case "package_install": self = .packageInstall(spec: j.str("spec"), ok: j.flag("ok"))39 case "research_run": self = .run(VQResearchRun(json: j["run"] ?? .null))40 case "experiment": self = .experiment(VQLedgerEntry(json: j["experiment"] ?? .null))41 case "research_note": self = .note(VQResearchNote(json: j["note"] ?? .null))42 case "figure_lint": self = .figureLint(title: j.optStr("title"), ok: j.flag("ok"), issues: j.models("issues"), files: j["files"].flatMap { $0.objectValue == nil ? nil : .init(json: $0) })43 case "approval_request": self = .approvalRequest(VQResearchApproval(json: j["approval"] ?? .null))44 case "approval_resolved": self = .approvalResolved(VQResearchApproval(json: j["approval"] ?? .null))45 case "quality_gate": self = .qualityGate(passed: j.flag("passed"), findings: j.models("findings"))46 default: self = .other(type: type, payload: j)47 }48 }49}5051/// Everything the interface shows about a study. Views read this and nothing else.52public struct VQResearchState: Sendable, Equatable {53 public struct PlanStep: Sendable, Hashable {54 public enum Status: String, Sendable, Hashable { case pending, inProgress = "in_progress", done }55 public var title: String56 public var status: Status57 /// The steps of a `plan` array (`[{ title, status }]`); entries without a title are dropped.58 public static func list(_ json: VQJSON?) -> [PlanStep] {59 (json?.arrayValue ?? []).compactMap { st in st.optStr("title").map { PlanStep(title: $0, status: Status(rawValue: st.str("status")) ?? .pending) } }60 }61 }62 /// The linter's verdict on a figure. `fixed` is the ordinary case (nearly every figure gets an automatic contrast63 /// repair): show it quietly. `issues` = a warning or an error remains.64 public enum LintStatus: String, Sendable, Hashable, CaseIterable { case passed, fixed, issues }65 public struct Dataset: Sendable, Hashable, Identifiable {66 public var path: String67 public var rows: Int?68 public var columns: Int?69 public var bytes: Int?70 public var id: String { path }71 }72 public struct FileChange: Sendable, Hashable {73 public enum Action: String, Sendable, Hashable, CaseIterable { case write, patch, delete, library }74 public var action: VQOpen<Action>75 public var path: String76 public var bytes: Int?77 public var agent: String?78 }79 /// One notebook cell the agent executed — successful or not: failed ones are kept.80 public struct Execution: Sendable, Hashable, Identifiable {81 /// Stable inside the view: the ledger id when the state comes from REST, a local counter (`x12`) from a stream.82 public var key: String83 /// Ledger id.84 public var ledgerId: String?85 public var seq: Int?86 public var agent: String87 public var title: String?88 public var codeHash: String?89 /// Workspace path of the code (`logs/cells/<hash>.py`).90 public var codePath: String?91 /// The code itself when the stream carried it; otherwise read `codePath`.92 public var code: String?93 public var lines: Int?94 public var status: VQLedgerStatus95 public var durationMs: Double?96 public var error: String?97 /// Console output; nil = not kept.98 public var output: String?99 public var envHash: String?100 public var generationId: String?101 public var startedAt: Date?102 var toolId: String?103 public var figureKeys: [String]104 public var id: String { key }105 /// The stream cut the code short (it ends with "…"): read the whole cell from `codePath`.106 public var codeIsTruncated: Bool { code?.hasSuffix("…") ?? false }107 }108 public struct Install: Sendable, Hashable, Identifiable {109 public var key: String110 public var spec: String111 public var ok: Bool112 public var id: String { key }113 }114 public struct LintIssue: Sendable, Hashable, VQJSONDecodable {115 public var code: String116 public var severity: String117 public var message: String118 /// The linter repaired it.119 public var fixed: Bool120 public init(json j: VQJSON) { code = j.str("code"); severity = j.str("severity", "info"); message = j.str("message"); fixed = j.flag("fixed") }121 }122 public struct FigureFiles: Sendable, Hashable, VQJSONDecodable {123 public var png: String?, svg: String?, pdf: String?124 public init(png: String? = nil, svg: String? = nil, pdf: String? = nil) { self.png = png; self.svg = svg; self.pdf = pdf }125 public init(json j: VQJSON) { png = j.optStr("png"); svg = j.optStr("svg"); pdf = j.optStr("pdf") }126 }127 public struct Figure: Sendable, Hashable, Identifiable {128 public var key: String129 public var title: String?130 /// The linter's verdict; nil when it was not kept (a study reloaded after its stream expired).131 public var lintPassed: Bool?132 public var issues: [LintIssue]133 /// Workspace paths of the exports (PNG, SVG, PDF) — read them with `VQResearchAgentClient.fileData`.134 public var files: FigureFiles?135 /// Server path of the image as served to the report (`/figures/….png`).136 public var url: String?137 public var executionKey: String?138 public var generationId: String?139 /// Server id of the figure (new servers).140 public var figureId: String? = nil141 /// The title drawn on the figure: the text a report puts under it.142 public var caption: String? = nil143 /// passed | fixed | issues; nil when the verdict was not kept (studies recorded before the server stored it).144 public var lintStatus: VQOpen<LintStatus>? = nil145 /// Repaired automatically.146 public var fixes: [LintIssue] = []147 /// Left as they are (information-level ones included).148 public var findings: [LintIssue] = []149 public var at: Date? = nil150 var toolId: String? = nil151 public var id: String { key }152 public var openIssues: [LintIssue] { issues.filter { !$0.fixed && $0.severity != "info" } }153 }154 public struct QualityGate: Sendable, Hashable {155 public var generationId: String?156 public var passed: Bool157 public var findings: [VQQualityGateFinding]158 }159 public struct BudgetHit: Sendable, Hashable {160 public var kind: String161 public var spent: Double?162 public var limit: Double?163 }164 /// One entry of the activity feed (the decisions timeline). It points at an entity by key, so an update shows in place.165 public struct TimelineItem: Sendable, Hashable, Identifiable {166 public enum Kind: String, Sendable, Hashable, CaseIterable { case workspace, dataset, plan, file, execution, install, run, experiment, note, figure, approval, gate, budget }167 /// `<kind>:<ref>`168 public var key: String169 public var kind: Kind170 public var ref: String171 /// When the server said it; nil when the stream gave no time (never invented).172 public var at: Date?173 public var generationId: String?174 public var id: String { key }175 }176177 /// Highest stream index applied (−1 before the first). Events at or below it are ignored: replay from N is safe.178 public internal(set) var lastIndex = -1179 public internal(set) var workspaceId: String?180 public internal(set) var workspaceKind: String?181 public internal(set) var generationId: String?182 public internal(set) var sessionId: String?183 /// The question of the study.184 public internal(set) var objective: String?185 public internal(set) var plan: [PlanStep] = []186 /// How many times the agent rewrote its plan (new titles, not progress).187 public internal(set) var planRevisions = 0188 public internal(set) var datasets: [Dataset] = []189 public internal(set) var files: [FileChange] = []190 public internal(set) var executions: [Execution] = []191 public internal(set) var installs: [Install] = []192 public internal(set) var runs: [VQResearchRun] = []193 /// Declared experiments (failed, inconclusive and abandoned ones included).194 public internal(set) var experiments: [VQLedgerEntry] = []195 public internal(set) var notes: [VQResearchNote] = []196 public internal(set) var figures: [Figure] = []197 public internal(set) var approvals: [VQResearchApproval] = []198 public internal(set) var gates: [QualityGate] = []199 public internal(set) var budgetHits: [BudgetHit] = []200 public internal(set) var timeline: [TimelineItem] = []201 /// The limits the generation runs under (`budget_applied`, or the last generation's from REST).202 public internal(set) var budgetApplied: VQBudgetApplied?203 /// The analyses of the workspace (REST only), oldest first: `interrupted` ones were cut by a server restart.204 public internal(set) var generations: [VQResearchGeneration] = []205 /// The answer as written so far (`text` events), or the stored answer of a finished study.206 public internal(set) var report = ""207 /// The generation ended (done or error).208 public internal(set) var finished = false209 var counter = 0210 var lastCompletedToolId: String?211212 public init() {}213214 public var pendingApprovals: [VQResearchApproval] { approvals.filter { $0.status == .pending } }215 /// Findings of every quality gate, open ones first.216 public var gateFindings: [VQQualityGateFinding] { gates.flatMap(\.findings).sorted { ($0.isOpen ? 0 : 1) < ($1.isOpen ? 0 : 1) } }217 public var isEmpty: Bool { workspaceId == nil && executions.isEmpty && runs.isEmpty && experiments.isEmpty && approvals.isEmpty && datasets.isEmpty }218 public func execution(_ key: String) -> Execution? { executions.first { $0.key == key } }219 public func figure(_ key: String) -> Figure? { figures.first { $0.key == key } }220221 public struct Counts: Sendable, Equatable {222 /// Notebook cells + script runs (what the ledger calls executions).223 public var executions: Int224 /// Of which did not end well (failed, timeout, killed, interrupted, cancelled).225 public var failed: Int226 public var cells: Int227 public var running: Int228 public var runs: Int229 public var runsRunning: Int230 public var experiments: Int231 public var figures: Int232 public var installs: Int233 public var pendingApprovals: Int234 }235 public var counts: Counts {236 Counts(executions: executions.count + runs.count,237 failed: executions.filter { $0.status != .ok && $0.status != .running }.count + runs.filter { $0.status != .done && $0.status != .running }.count,238 cells: executions.count, running: executions.filter { $0.status == .running }.count, runs: runs.count, runsRunning: runs.filter(\.isRunning).count,239 experiments: experiments.count, figures: figures.count, installs: installs.count, pendingApprovals: pendingApprovals.count)240 }241 /// The state restricted to one generation (a workspace can hold several studies of a conversation).242 public func filtered(generationId g: String) -> VQResearchState {243 var s = self244 s.executions = executions.filter { $0.generationId == g }245 s.runs = runs.filter { $0.generationId == g }246 s.experiments = experiments.filter { $0.generationId == g }247 s.figures = figures.filter { $0.generationId == g }248 s.gates = gates.filter { $0.generationId == g }249 s.timeline = timeline.filter { $0.generationId == g || $0.generationId == nil }250 return s251 }252}253254public enum VQResearchReducer {255 public static let notebookTool = "execute_custom_python_analysis"256257 /// Applies one stream event (its JSON). `index` is the SSE id when the transport knows it: an event at or below258 /// the highest index already applied is a replay and is ignored. Duplicates of entities that have an id are259 /// upserts; an event type this version does not know leaves the state untouched.260 public static func reduce(_ state: inout VQResearchState, _ event: VQJSON, index: Int? = nil) {261 guard event["type"]?.stringValue != nil else { return }262 if let index, index <= state.lastIndex { return }263 apply(&state, event)264 if let index { state.lastIndex = index }265 }266267 public static func reduce(_ state: inout VQResearchState, _ event: VQSSEEvent) { reduce(&state, event.json, index: event.id) }268269 /// Reduces a list of events on top of `state`. `firstIndex` numbers them like the SSE ids.270 public static func replay(_ events: [VQJSON], onto state: VQResearchState = VQResearchState(), firstIndex: Int? = nil) -> VQResearchState {271 var s = state272 for (i, e) in events.enumerated() { reduce(&s, e, index: firstIndex.map { $0 + i }) }273 return s274 }275276 /// Verdict, repairs and remaining findings of a `figure_lint` payload or a stored figure, old shape or new.277 static func lint(status: String?, ok: Bool?, issues: [VQResearchState.LintIssue], fixes: [VQResearchState.LintIssue]?, findings: [VQResearchState.LintIssue]?)278 -> (status: VQOpen<VQResearchState.LintStatus>?, passed: Bool?, fixes: [VQResearchState.LintIssue], findings: [VQResearchState.LintIssue]) {279 let fx = fixes ?? issues.filter(\.fixed), fd = findings ?? issues.filter { !$0.fixed }280 if let status { let st = VQOpen<VQResearchState.LintStatus>(status); return (st, st.known.map { $0 != .issues } ?? ok, fx, fd) }281 guard let ok else { return (nil, nil, fx, fd) }282 return (.known(!ok ? .issues : fx.isEmpty ? .passed : .fixed), ok, fx, fd)283 }284285 private static func push(_ s: inout VQResearchState, _ kind: VQResearchState.TimelineItem.Kind, _ ref: String, at: Date?) {286 let key = "\(kind.rawValue):\(ref)"287 if s.timeline.contains(where: { $0.key == key }) { return }288 s.timeline.append(.init(key: key, kind: kind, ref: ref, at: at, generationId: s.generationId))289 }290291 static func executionStatus(ok: Bool, error: String?) -> VQLedgerStatus {292 if ok { return .ok }293 if let error, error.contains("time limit") || error.contains("Execution stopped after") { return .timeout }294 if let error, error.contains("stopped by the sandbox") { return .killed }295 return .failed296 }297298 /// Text a person can read out of the notebook's tool result (the stream sends the object the model received).299 public static func output(ofToolResult result: VQJSON?) -> String? {300 guard let result else { return nil }301 var text: String302 if let s = result.stringValue { text = s }303 else if result.objectValue != nil {304 var parts: [String] = []305 if let o = result.optStr("output"), !o.isEmpty { parts.append(o) }306 if let r = result["result"], r.objectValue == nil, r.arrayValue == nil, let t = r.displayString { parts.append(t) }307 text = parts.joined(separator: "\n")308 } else { return nil }309 if let r = text.range(of: "\n*\\[Notebook session — variables in memory:[^\\]]*\\]\\s*$", options: .regularExpression) { text.removeSubrange(r) }310 if let r = text.range(of: "\n---\nFIGURES PRODUCED BY THIS CALL") { text = String(text[..<r.lowerBound]) }311 text = text.trimmingCharacters(in: .whitespacesAndNewlines)312 return text.isEmpty ? nil : text313 }314315 private static func merge(run raw: VQJSON, into prev: VQResearchRun?, generationId: String?) -> VQResearchRun {316 var next = VQResearchRun(json: raw)317 guard let prev else { if next.generationId == nil { next.generationId = generationId }; return next }318 // A late duplicate of the start event must not bring a finished run back to life.319 let regress = (prev.status.known?.isTerminal ?? true) && next.status == .running320 var out = prev321 out.label = next.label ?? prev.label; out.code = next.code ?? prev.code; out.codeHash = next.codeHash ?? prev.codeHash322 if raw["args"]?.arrayValue != nil { out.args = next.args }323 if next.network == .open { out.network = .open }324 out.startedAt = next.startedAt ?? prev.startedAt325 out.generationId = next.generationId ?? prev.generationId ?? generationId326 if !regress {327 out.status = raw["status"]?.stringValue == nil ? prev.status : next.status328 out.progress = next.progress ?? prev.progress; out.exitCode = next.exitCode ?? prev.exitCode; out.error = next.error ?? prev.error; out.finishedAt = next.finishedAt ?? prev.finishedAt329 }330 return out331 }332333 private static func merge(experiment raw: VQJSON, into prev: VQLedgerEntry?, generationId: String?) -> VQLedgerEntry {334 var next = VQLedgerEntry(json: raw)335 if let prev {336 next.seq = next.seq ?? prev.seq; next.title = next.title ?? prev.title; next.hypothesis = next.hypothesis ?? prev.hypothesis; next.method = next.method ?? prev.method337 if raw["status"]?.stringValue == nil { next.status = prev.status }338 next.metrics = next.metrics ?? prev.metrics; next.diagnostics = next.diagnostics ?? prev.diagnostics339 if raw["artifacts"]?.arrayValue == nil { next.artifacts = prev.artifacts }340 next.codeRef = next.codeRef ?? prev.codeRef; next.runId = next.runId ?? prev.runId; next.notes = next.notes ?? prev.notes; next.error = next.error ?? prev.error341 next.envHash = next.envHash ?? prev.envHash; next.codeHash = next.codeHash ?? prev.codeHash; next.startedAt = next.startedAt ?? prev.startedAt; next.finishedAt = next.finishedAt ?? prev.finishedAt; next.durationMs = next.durationMs ?? prev.durationMs342 next.generationId = next.generationId ?? prev.generationId343 }344 if next.generationId == nil { next.generationId = generationId }345 return next346 }347348 private static func apply(_ s: inout VQResearchState, _ e: VQJSON) {349 let at = e.date("at") // the server time every event of a generation carries (new servers)350 switch e.str("type") {351 case "budget_applied":352 s.budgetApplied = VQBudgetApplied(json: e)353 case "code_execution_start":354 // a cell starts running: the same toolId as its tool_start (which carries the code) and as its end355 let toolId = e.optStr("toolId")356 if let toolId, let i = s.executions.firstIndex(where: { $0.toolId == toolId }) {357 s.executions[i].title = e.optStr("description") ?? s.executions[i].title; s.executions[i].codeHash = e.optStr("codeHash") ?? s.executions[i].codeHash358 s.executions[i].lines = e.int("lines") ?? s.executions[i].lines; s.executions[i].startedAt = at ?? s.executions[i].startedAt359 s.executions[i].agent = e.str("agent", s.executions[i].agent)360 if let hash = s.executions[i].codeHash, s.executions[i].codePath == nil { s.executions[i].codePath = "logs/cells/\(hash).py" }361 } else {362 s.counter += 1363 let key = "x\(s.counter)"364 let hash = e.optStr("codeHash")365 s.executions.append(.init(key: key, ledgerId: nil, seq: nil, agent: e.str("agent", "lead"), title: e.optStr("description"), codeHash: hash, codePath: hash.map { "logs/cells/\($0).py" }, code: nil,366 lines: e.int("lines"), status: .running, durationMs: nil, error: nil, output: nil, envHash: nil, generationId: s.generationId, startedAt: at, toolId: toolId, figureKeys: []))367 push(&s, .execution, key, at: at)368 }369 case "job":370 s.generationId = e.optStr("id") ?? s.generationId; s.sessionId = e.optStr("sessionId") ?? s.sessionId; s.objective = e.optStr("query") ?? s.objective371 case "text":372 if let chunk = e.optStr("content") { s.report += chunk }373 case "research_workspace":374 guard let id = e.optStr("workspaceId"), s.workspaceId != id else { return }375 s.workspaceId = id; s.workspaceKind = e.str("kind", "session")376 push(&s, .workspace, id, at: at)377 case "plan":378 let steps: [VQResearchState.PlanStep] = e.list("steps").compactMap { step in379 guard let title = step.optStr("title") else { return nil }380 return .init(title: title, status: VQResearchState.PlanStep.Status(rawValue: step.str("status")) ?? .pending)381 }382 guard steps != s.plan else { return }383 // New titles are a methodological decision (the agent re-planned); a status change is only progress.384 let retitled = !s.plan.isEmpty && steps.map(\.title) != s.plan.map(\.title)385 if s.plan.isEmpty { s.planRevisions = 0; push(&s, .plan, "0", at: at) }386 else if retitled { s.planRevisions += 1; push(&s, .plan, String(s.planRevisions), at: at) }387 s.plan = steps388 case "dataset":389 guard let path = e.optStr("path") else { return }390 let d = VQResearchState.Dataset(path: path, rows: e.int("rows"), columns: e.int("columns"), bytes: e.int("bytes"))391 if let i = s.datasets.firstIndex(where: { $0.path == path }) { s.datasets[i] = d } else { s.datasets.append(d) }392 push(&s, .dataset, path, at: at)393 case "research_file":394 guard let path = e.optStr("path") else { return }395 s.counter += 1396 s.files.append(.init(action: e.open("action", .write), path: path, bytes: e.int("bytes"), agent: e.optStr("agent")))397 push(&s, .file, "\(s.counter):\(path)", at: at)398 case "tool_start":399 guard e.str("tool") == notebookTool else { return }400 let toolId = e.optStr("toolId")401 if let toolId, s.executions.contains(where: { $0.toolId == toolId }) { return }402 let input = e["input"] ?? .null403 s.counter += 1404 let key = "x\(s.counter)"405 let code = input.optStr("code")406 s.executions.append(.init(key: key, ledgerId: nil, seq: nil, agent: "lead", title: input.optStr("description"), codeHash: nil, codePath: nil, code: code,407 lines: code.map { $0.components(separatedBy: "\n").count }, status: .running, durationMs: nil, error: nil, output: nil, envHash: nil,408 generationId: s.generationId, startedAt: at, toolId: toolId, figureKeys: []))409 push(&s, .execution, key, at: at)410 case "code_execution":411 let id = e.optStr("experimentId")412 if let id, s.executions.contains(where: { $0.ledgerId == id }) { return }413 let agent = e.str("agent", "lead"), title = e.optStr("description"), error = e.optStr("error")414 let status = executionStatus(ok: e.flag("ok"), error: error)415 // The cell announced by tool_start: same description first (parallel cells), else the oldest one still running.416 let running = s.executions.indices.filter { s.executions[$0].status == .running && s.executions[$0].toolId != nil && agent == "lead" }417 // New servers name the call (`toolId`): pair exactly. Older streams: by description, else the oldest running.418 let byTool = e.optStr("toolId").flatMap { t in s.executions.firstIndex { $0.toolId == t } }419 if let i = byTool ?? (e.optStr("toolId") == nil ? (running.first(where: { s.executions[$0].title == title }) ?? running.first) : nil) {420 if s.executions[i].status != .running, s.executions[i].ledgerId != nil { return } // the end of this call was already applied421 s.executions[i].ledgerId = id; s.executions[i].agent = agent; s.executions[i].title = title ?? s.executions[i].title422 s.executions[i].codeHash = e.optStr("codeHash"); s.executions[i].codePath = e.optStr("code"); s.executions[i].lines = e.int("lines") ?? s.executions[i].lines423 s.executions[i].status = status; s.executions[i].durationMs = e.num("durationMs"); s.executions[i].error = error424 } else {425 s.counter += 1426 let key = "x\(s.counter)"427 s.executions.append(.init(key: key, ledgerId: id, seq: nil, agent: agent, title: title, codeHash: e.optStr("codeHash"), codePath: e.optStr("code"), code: nil, lines: e.int("lines"),428 status: status, durationMs: e.num("durationMs"), error: error, output: nil, envHash: nil, generationId: s.generationId,429 startedAt: at.flatMap { t in e.num("durationMs").map { t.addingTimeInterval(-$0 / 1000) } }, toolId: e.optStr("toolId"), figureKeys: []))430 push(&s, .execution, key, at: at)431 }432 case "tool_complete":433 s.lastCompletedToolId = e.str("tool") == notebookTool ? e.optStr("toolId") : nil434 case "tool_result":435 // The result follows the tool_complete of the same call; it carries no id of its own.436 guard e.str("tool") == notebookTool, let toolId = e.optStr("toolId") ?? s.lastCompletedToolId else { return }437 s.lastCompletedToolId = nil438 if let i = s.executions.firstIndex(where: { $0.toolId == toolId }), s.executions[i].output == nil { s.executions[i].output = output(ofToolResult: e["result"]) }439 case "tool_error":440 guard e.str("tool") == notebookTool, let toolId = e.optStr("toolId"), let i = s.executions.firstIndex(where: { $0.toolId == toolId }) else { return }441 let error = e.optStr("error") ?? s.executions[i].error ?? "Error"442 // The tool error is the full text; the one of code_execution is cut short by the server.443 if s.executions[i].status != .running, (s.executions[i].error?.count ?? 0) >= error.count { return }444 if s.executions[i].status == .running || s.executions[i].status == .ok { s.executions[i].status = executionStatus(ok: false, error: error) }445 s.executions[i].error = error446 case "package_install":447 guard let spec = e.optStr("spec") else { return }448 s.counter += 1449 let key = "i\(s.counter)"450 s.installs.append(.init(key: key, spec: spec, ok: e.flag("ok")))451 push(&s, .install, key, at: at)452 case "research_run":453 guard let raw = e["run"], let id = raw.optStr("id") else { return }454 if let i = s.runs.firstIndex(where: { $0.id == id }) { s.runs[i] = merge(run: raw, into: s.runs[i], generationId: s.generationId) }455 else { let run = merge(run: raw, into: nil, generationId: s.generationId); s.runs.append(run); push(&s, .run, id, at: run.startedAt ?? at) }456 case "experiment":457 guard let raw = e["experiment"], let id = raw.optStr("id") else { return }458 // The stream may also carry automatic `execution` entries: those are already shown as cells and runs.459 if raw.str("kind") == "execution" { return }460 if let i = s.experiments.firstIndex(where: { $0.id == id }) { s.experiments[i] = merge(experiment: raw, into: s.experiments[i], generationId: s.generationId) }461 else { let x = merge(experiment: raw, into: nil, generationId: s.generationId); s.experiments.append(x); push(&s, .experiment, id, at: x.startedAt ?? at) }462 case "research_note":463 guard let raw = e["note"], raw["id"] != nil else { return }464 let note = VQResearchNote(json: raw)465 if let i = s.notes.firstIndex(where: { $0.id == note.id }) { s.notes[i] = note } else { s.notes.append(note) }466 push(&s, .note, note.id, at: note.createdAt ?? at)467 case "figure_lint":468 let files = e["files"].flatMap { $0.objectValue == nil ? nil : VQResearchState.FigureFiles(json: $0) }469 let figureId = e.optStr("figureId"), url = e.optStr("url")470 let v = lint(status: e.optStr("status"), ok: e.optFlag("ok"), issues: e.models("issues"), fixes: e["fixes"]?.arrayValue.map { $0.map(VQResearchState.LintIssue.init(json:)) },471 findings: e["findings"]?.arrayValue.map { $0.map(VQResearchState.LintIssue.init(json:)) })472 // A figure is identified by its server id, else its PNG in the workspace, else its address; a replay or473 // the image event arriving first must not create it twice.474 let existing = s.figures.firstIndex { f in (figureId != nil && f.figureId == figureId) || (files?.png != nil && f.files?.png == files?.png) || (url != nil && f.url == url) }475 if let i = existing {476 guard s.figures[i].lintStatus == nil else { return }477 s.figures[i].figureId = figureId; s.figures[i].title = e.optStr("title") ?? s.figures[i].title; s.figures[i].caption = e.optStr("caption")478 s.figures[i].lintStatus = v.status; s.figures[i].lintPassed = v.passed; s.figures[i].issues = e.models("issues"); s.figures[i].fixes = v.fixes; s.figures[i].findings = v.findings479 s.figures[i].files = files ?? s.figures[i].files; s.figures[i].at = at ?? s.figures[i].at480 return481 }482 s.counter += 1483 let key = "f\(s.counter)"484 let owner = e.optStr("toolId").flatMap { t in s.executions.firstIndex { $0.toolId == t } } ?? s.executions.lastIndex { $0.status != .running }485 var figure = VQResearchState.Figure(key: key, title: e.optStr("title"), lintPassed: v.passed, issues: e.models("issues"), files: files, url: url,486 executionKey: owner.map { s.executions[$0].key }, generationId: s.generationId)487 figure.figureId = figureId; figure.caption = e.optStr("caption"); figure.lintStatus = v.status; figure.fixes = v.fixes; figure.findings = v.findings; figure.at = at; figure.toolId = e.optStr("toolId")488 s.figures.append(figure)489 if let owner { s.executions[owner].figureKeys.append(key) }490 push(&s, .figure, key, at: at)491 case "custom_python_figures":492 let urls = e.strings("figureUrls")493 if urls.isEmpty || urls.allSatisfy({ u in s.figures.contains { $0.url == u } }) { return }494 // The images of a cell arrive after its lint reports, in the same order: give each report its address.495 let waiting = s.figures.indices.filter { s.figures[$0].url == nil && s.figures[$0].lintPassed != nil }496 let tail = Array(waiting.suffix(urls.count))497 for (k, url) in urls.enumerated() {498 if tail.count == urls.count { s.figures[tail[k]].url = url; continue }499 // A figure the linter did not report (a server without the visual check).500 s.counter += 1501 let key = "f\(s.counter)"502 let owner = s.executions.lastIndex { $0.status != .running }503 s.figures.append(.init(key: key, title: e.optStr("description"), lintPassed: nil, issues: [], files: nil, url: url, executionKey: owner.map { s.executions[$0].key }, generationId: s.generationId))504 if let owner { s.executions[owner].figureKeys.append(key) }505 push(&s, .figure, key, at: at)506 }507 case "approval_request", "approval_resolved":508 guard let raw = e["approval"], let id = raw.optStr("id") else { return }509 var approval = VQResearchApproval(json: raw)510 if let i = s.approvals.firstIndex(where: { $0.id == id }) {511 let prev = s.approvals[i]512 // A request replayed after its resolution must not reopen it.513 if e.str("type") == "approval_request", prev.status != .pending { return }514 approval.reason = approval.reason ?? prev.reason; approval.detail = approval.detail ?? prev.detail515 approval.createdAt = approval.createdAt ?? prev.createdAt; approval.resolvedAt = approval.resolvedAt ?? prev.resolvedAt516 if raw["action"]?.stringValue == nil { approval.action = prev.action }517 if raw["status"]?.stringValue == nil { approval.status = prev.status }518 s.approvals[i] = approval519 } else { s.approvals.append(approval); push(&s, .approval, id, at: approval.createdAt ?? at) }520 case "quality_gate":521 let gate = VQResearchState.QualityGate(generationId: s.generationId, passed: e.flag("passed"), findings: e.models("findings"))522 if let i = s.gates.firstIndex(where: { $0.generationId == gate.generationId }) { s.gates[i] = gate } else { s.gates.append(gate) }523 push(&s, .gate, gate.generationId ?? "0", at: at)524 case "budget":525 let hit = VQResearchState.BudgetHit(kind: e.str("kind", "cost"), spent: e.num("spent"), limit: e.num("limit"))526 if s.budgetHits.contains(where: { $0.kind == hit.kind && $0.limit == hit.limit }) { return }527 s.budgetHits.append(hit)528 push(&s, .budget, "\(hit.kind):\(hit.limit.map { VQJSON.number($0).text() } ?? "")", at: at)529 case "done", "error":530 guard !s.finished else { return }531 s.finished = true532 // Nothing can still be running in a generation that ended: a cell left open was cut with it.533 for i in s.executions.indices where s.executions[i].status == .running {534 s.executions[i].status = .interrupted535 s.executions[i].error = s.executions[i].error ?? "Interrupted with the analysis."536 }537 default:538 return539 }540 }541}542543// MARK: rebuilding a finished study from REST544545public extension VQResearchState {546 /// Groups `artifacts/figures/<stem>.{png,svg,pdf}` into one figure per stem, in file-name (chronological) order.547 static func figureFiles(in paths: [String]) -> [(stem: String, files: FigureFiles)] {548 let dir = "artifacts/figures/"549 var byStem: [String: FigureFiles] = [:]550 for p in paths where p.hasPrefix(dir) {551 guard let dot = p.lastIndex(of: ".") else { continue }552 let stem = String(p[..<dot]), ext = p[p.index(after: dot)...].lowercased()553 var f = byStem[stem] ?? FigureFiles()554 switch ext {555 case "png": f.png = p556 case "svg": f.svg = p557 case "pdf": f.pdf = p558 default: continue559 }560 byStem[stem] = f561 }562 return byStem.filter { $0.value.png != nil || $0.value.svg != nil }.sorted { $0.key < $1.key }.map { ($0.key, $0.value) }563 }564565 private static func title(fromStem stem: String) -> String? {566 var name = String(stem.dropFirst("artifacts/figures/".count))567 if let r = name.range(of: "^\\d{8}-\\d{6}-\\d+-", options: .regularExpression) { name.removeSubrange(r) }568 name = name.replacingOccurrences(of: "-", with: " ").trimmingCharacters(in: .whitespaces)569 return name.isEmpty ? nil : name570 }571572 /// The same view model as the stream reducer, rebuilt from what the server keeps once a generation is over: the573 /// workspace detail (ledger, runs, notes, files, approvals) and the stored conversation (`GET /api/sessions/:id`:574 /// console outputs, the outcome of the quality gate, the answer). What the server does not keep is left empty575 /// rather than guessed: the linter verdict of each figure (`lintPassed == nil`), the plan (taken from the stored576 /// `update_plan` call when there is one), the times of events that carried none.577 init(detail: VQResearchSessionDetail, conversation: VQJSON? = nil, approvals extraApprovals: [VQResearchApproval] = []) {578 self.init()579 let wsId = detail.session.id580 workspaceId = wsId; workspaceKind = detail.session.kind.rawValue581 sessionId = detail.session.sessionIds.last582 generations = detail.generations583 finished = !detail.generations.contains { $0.status == .running }584 // New servers keep the plan and the limits of each generation; older studies have neither (left empty).585 plan = detail.plan.isEmpty ? (detail.generations.last { !$0.plan.isEmpty }?.plan ?? []) : detail.plan586 budgetApplied = detail.generations.last?.budgetApplied587 for g in detail.generations { for b in g.budget { budgetHits.append(.init(kind: b.kind.rawValue, spent: b.spent, limit: b.limit)) } }588 let messages = conversation.map(VQMessageExtras.messages(of:)) ?? []589 let mine = messages.filter { m in m["research"]?.objectValue == nil || m["research"]?.optStr("workspaceId") == wsId }590 objective = mine.first?.optStr("question") ?? detail.session.title591 generationId = mine.last?.optStr("generationId")592 report = mine.last.map { m in593 // the stored answer is JSON text of a string in the legacy route; a plain string otherwise594 let raw = m.str("answer")595 return (try? VQJSON.parse(raw))?.stringValue ?? raw596 }?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""597598 func toolResults(_ m: VQJSON) -> [VQJSON] {599 if let a = m["toolResults"]?.arrayValue { return a }600 if let s = m["toolResults"]?.stringValue { return (try? VQJSON.parse(s))?.arrayValue ?? [] }601 return []602 }603 // Console outputs: the stored conversation keeps the results of the successful notebook calls of a generation,604 // in order. They are attached only when the count matches the ledger exactly — a wrong pairing would be worse605 // than no output.606 var outputs: [String: String] = [:]607 for m in mine {608 guard let gen = m.optStr("generationId") else { continue }609 let results = toolResults(m).filter { $0.str("tool") == VQResearchReducer.notebookTool }610 let cells = detail.executions.filter { $0.generationId == gen && $0.runId == nil && $0.status == .ok }.sorted { ($0.seq ?? 0) < ($1.seq ?? 0) }611 guard !results.isEmpty, results.count == cells.count else { continue }612 for (c, r) in zip(cells, results) { if let o = VQResearchReducer.output(ofToolResult: r["result"]) { outputs[c.id] = o } }613 // the last plan the agent kept614 if plan.isEmpty, let planCall = toolResults(m).last(where: { $0.str("tool") == "update_plan" }) { plan = PlanStep.list(planCall["result"]?["steps"] ?? planCall["result"]?["plan"]) }615 }616617 // Figures the server kept with their verdict (new servers), then — for studies recorded before that — the618 // image files of the workspace no record covers (verdict unknown).619 var figs: [Figure] = detail.figures.map { r in620 let v = VQResearchReducer.lint(status: r.status?.rawValue, ok: nil, issues: r.fixes + r.findings, fixes: r.fixes, findings: r.findings)621 var f = Figure(key: "f:\(r.id)", title: r.title ?? r.caption, lintPassed: v.passed, issues: r.fixes + r.findings, files: r.files, url: r.url, executionKey: nil, generationId: r.generationId)622 f.figureId = r.id; f.caption = r.caption; f.lintStatus = v.status; f.fixes = r.fixes; f.findings = r.findings; f.at = r.at; f.toolId = r.toolId623 if let hash = r.codeHash, let owner = detail.executions.first(where: { $0.codeHash == hash && $0.generationId == r.generationId && $0.runId == nil }) { f.executionKey = owner.id }624 return f625 }626 let recorded = Set(figs.compactMap { $0.files?.png })627 figs += Self.figureFiles(in: detail.files.filter { !$0.isDirectory }.map(\.path)).filter { !($0.files.png.map(recorded.contains) ?? false) }.map {628 Figure(key: "f:\($0.stem)", title: nil, lintPassed: nil, issues: [], files: $0.files, url: nil, executionKey: nil, generationId: nil)629 }630 var line: [TimelineItem] = []631 for x in detail.executions.sorted(by: { ($0.seq ?? 0) < ($1.seq ?? 0) }) where x.runId == nil { // script runs are shown from `runs`632 let title = x.title == "Notebook cell" ? nil : x.title633 var figureKeys: [String] = []634 for path in x.artifacts {635 guard let i = figs.firstIndex(where: { $0.files?.png == path }) else { continue }636 figs[i].executionKey = x.id; figs[i].generationId = figs[i].generationId ?? x.generationId; figs[i].title = figs[i].title ?? title637 figureKeys.append(figs[i].key)638 }639 for i in figs.indices where figs[i].executionKey == x.id && !figureKeys.contains(figs[i].key) { figureKeys.append(figs[i].key) }640 executions.append(.init(key: x.id, ledgerId: x.id, seq: x.seq, agent: x.codeRef.flatMap { $0.hasPrefix("notebook:") ? String($0.dropFirst(9)) : nil } ?? "lead", title: title,641 codeHash: x.codeHash, codePath: x.codeHash.map { "logs/cells/\($0).py" }, code: nil, lines: nil,642 status: x.status.known.flatMap { [.ok, .failed, .timeout, .killed, .interrupted, .running].contains($0) ? $0 : nil } ?? .failed,643 durationMs: x.durationMs, error: x.error, output: outputs[x.id], envHash: x.envHash, generationId: x.generationId, startedAt: x.startedAt, toolId: nil, figureKeys: figureKeys))644 line.append(.init(key: "execution:\(x.id)", kind: .execution, ref: x.id, at: x.startedAt, generationId: x.generationId))645 }646 // Figures no ledger entry claims: the stored conversation lists the figures of each generation in order.647 for m in mine {648 guard let gen = m.optStr("generationId") else { continue }649 let stored = m["customPythonFigures"]?.arrayValue ?? []650 let unowned = figs.indices.filter { figs[$0].generationId == nil }651 let ofGen = figs.indices.filter { figs[$0].generationId == gen }652 if ofGen.isEmpty, !stored.isEmpty, unowned.count >= stored.count {653 for (k, i) in unowned.prefix(stored.count).enumerated() {654 figs[i].generationId = gen655 figs[i].title = stored[k].optStr("description") ?? figs[i].title656 figs[i].url = stored[k].strings("figureUrls").first657 }658 }659 }660 for i in figs.indices {661 if figs[i].title == nil { figs[i].title = Self.title(fromStem: String(figs[i].key.dropFirst(2))) }662 let owner = executions.first { $0.key == figs[i].executionKey }663 line.append(.init(key: "figure:\(figs[i].key)", kind: .figure, ref: figs[i].key, at: figs[i].at ?? owner?.startedAt.map { $0.addingTimeInterval((owner?.durationMs ?? 0) / 1000) }, generationId: figs[i].generationId))664 }665 figures = figs666 runs = detail.runs.sorted { ($0.startedAt ?? .distantPast) < ($1.startedAt ?? .distantPast) }667 for r in runs { line.append(.init(key: "run:\(r.id)", kind: .run, ref: r.id, at: r.startedAt, generationId: r.generationId)) }668 experiments = detail.experiments.sorted { ($0.seq ?? 0) < ($1.seq ?? 0) }669 for x in experiments { line.append(.init(key: "experiment:\(x.id)", kind: .experiment, ref: x.id, at: x.startedAt, generationId: x.generationId)) }670 notes = detail.notes671 for n in notes { line.append(.init(key: "note:\(n.id)", kind: .note, ref: n.id, at: n.createdAt, generationId: nil)) }672 approvals = detail.approvals + extraApprovals.filter { a in (a.workspaceId == nil || a.workspaceId == wsId) && !detail.approvals.contains { $0.id == a.id } }673 approvals.sort { ($0.createdAt ?? .distantPast) < ($1.createdAt ?? .distantPast) }674 if !plan.isEmpty { line.append(.init(key: "plan:0", kind: .plan, ref: "0", at: detail.generations.first { !$0.plan.isEmpty }?.startedAt, generationId: nil)) }675 for a in approvals { line.append(.init(key: "approval:\(a.id)", kind: .approval, ref: a.id, at: a.createdAt, generationId: nil)) }676 // Datasets: the files of data/ (their profile is read on demand through the inspect route).677 datasets = detail.files.filter { !$0.isDirectory && $0.path.hasPrefix("data/") }.map { Dataset(path: $0.path, rows: nil, columns: nil, bytes: $0.bytes) }678 for m in mine {679 guard let findings = m["research"]?["qualityGate"]?.arrayValue?.map(VQQualityGateFinding.init(json:)) else { continue }680 let gen = m.optStr("generationId")681 gates.append(.init(generationId: gen, passed: !findings.contains(where: \.isOpen), findings: findings))682 // The gate closes its generation: it sits after everything else that generation did.683 let last = line.filter { $0.generationId == gen }.compactMap(\.at).max()684 line.append(.init(key: "gate:\(gen ?? "0")", kind: .gate, ref: gen ?? "0", at: last?.addingTimeInterval(0.001), generationId: gen))685 }686 // Chronological; entries without a time keep their relative order at the end.687 timeline = line.enumerated().sorted { a, b in688 let x = a.element.at ?? .distantFuture, y = b.element.at ?? .distantFuture689 return x == y ? a.offset < b.offset : x < y690 }.map(\.element)691 timeline.insert(.init(key: "workspace:\(wsId)", kind: .workspace, ref: wsId, at: detail.session.createdAt, generationId: nil), at: 0)692 }693694 /// The view of a study that is running: the live state of its generation on top of what the workspace already695 /// held from earlier generations. Live entities win; the snapshot fills what the stream does not carry (the696 /// agent's notes on an experiment, the environment hash of an execution, times).697 func merged(over snapshot: VQResearchState?) -> VQResearchState {698 guard let snapshot else { return self }699 var out = self700 let gen = generationId701 let liveExec = Set(executions.compactMap(\.ledgerId)), liveRuns = Set(runs.map(\.id)), liveExp = Set(experiments.map(\.id)), liveNotes = Set(notes.map(\.id))702 let livePngs = Set(figures.compactMap { $0.files?.png })703 let snapExec = Dictionary(snapshot.executions.compactMap { x in x.ledgerId.map { ($0, x) } }, uniquingKeysWith: { a, _ in a })704 let snapExp = Dictionary(snapshot.experiments.map { ($0.id, $0) }, uniquingKeysWith: { a, _ in a })705 out.executions = snapshot.executions.filter { ($0.generationId != gen || gen == nil) && !($0.ledgerId.map(liveExec.contains) ?? false) } + executions.map { x in706 guard let s = x.ledgerId.flatMap({ snapExec[$0] }) else { return x }707 var y = x; y.seq = x.seq ?? s.seq; y.envHash = x.envHash ?? s.envHash; y.startedAt = x.startedAt ?? s.startedAt; y.output = x.output ?? s.output708 return y709 }710 out.runs = snapshot.runs.filter { !liveRuns.contains($0.id) } + runs711 out.experiments = snapshot.experiments.filter { !liveExp.contains($0.id) } + experiments.map { x in712 guard let s = snapExp[x.id] else { return x }713 var y = x; y.notes = x.notes ?? s.notes; y.envHash = x.envHash ?? s.envHash; y.codeHash = x.codeHash ?? s.codeHash714 return y715 }716 out.notes = snapshot.notes.filter { !liveNotes.contains($0.id) } + notes717 out.figures = snapshot.figures.filter { f in !(f.files?.png.map(livePngs.contains) ?? false) && (f.generationId != gen || gen == nil) } + figures718 out.gates = snapshot.gates.filter { $0.generationId != gen } + gates719 out.datasets = snapshot.datasets.filter { d in !datasets.contains { $0.path == d.path } } + datasets720 out.approvals = snapshot.approvals.filter { a in !approvals.contains { $0.id == a.id } } + approvals721 if out.plan.isEmpty { out.plan = snapshot.plan }722 out.budgetApplied = budgetApplied ?? snapshot.budgetApplied723 out.generations = snapshot.generations724 out.objective = objective ?? snapshot.objective725 out.workspaceId = workspaceId ?? snapshot.workspaceId; out.workspaceKind = workspaceKind ?? snapshot.workspaceKind726 let liveKeys = Set(timeline.map(\.key))727 out.timeline = snapshot.timeline.filter { !liveKeys.contains($0.key) && ($0.generationId != gen || gen == nil) } + timeline728 return out729 }730}731