Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXFeatureRuns/Tests/ORIXFeatureRunsTests/IntegrationTests.swift3// Description: Integration tests on real services in a temporary folder (real SQLite, real subprocess): run list paging and filters, retry recovery, dashboard and scheduler loading, the console fed by the store and the live hub, one render per screen, and the 50 000-line console measurement.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import AppKit8import Foundation9import ORIXAppUI10@_spi(Preview) @testable import ORIXFeatureRuns11import SwiftUI12import Testing1314@MainActor15private func seedMany(_ store: ORIXStore, count: Int) async throws {16 let base = Date(timeIntervalSince1970: 1_790_000_000)17 let statuses: [RunStatus] = [.succeeded, .failed, .canceled, .succeeded, .timedOut]18 for i in 0..<count {19 let start = base.addingTimeInterval(Double(-i) * 3600) // run 0 is the newest20 var r = RunRecord(id: String(format: "run-%03d", i), kind: i % 3 == 0 ? "workflow" : "agent", title: i % 10 == 0 ? "Invoice batch \(i)" : "Routine \(i)",21 trigger: i % 4 == 0 ? "cron" : "manual", mode: .guided, status: statuses[i % statuses.count], queuedAt: start, startedAt: start,22 endedAt: start.addingTimeInterval(30))23 if i == 7 { r.errorSummary = "the invoice folder is missing" }24 try await store.saveRun(r)25 }26}2728private func pngSize(_ url: URL) -> Int { (try? FileManager.default.attributesOfItem(atPath: url.path)[.size] as? Int) ?? 0 }2930@MainActor31private func render<V: View>(_ name: String, _ view: V, _ model: AppModel, size: CGSize = CGSize(width: 1280, height: 820)) async throws -> Int {32 let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("orix-runs-test-\(name).png")33 try await ShotRenderer.render(view.environment(model), size: size, to: url)34 defer { try? FileManager.default.removeItem(at: url) }35 return pngSize(url)36}3738@Suite("Integration · real store and services", .serialized)39@MainActor40struct RunsIntegrationTests {41 @Test func listPagesSearchesAndFilters() async throws {42 let model = try await PreviewEnvironment.make(seed: false)43 defer { try? FileManager.default.removeItem(at: model.services.paths.root) }44 try await seedMany(model.services.store, count: 150)45 let list = RunsListModel(store: model.services.store, pageSize: 40)4647 await list.reload()48 #expect(list.rows.count == 40 && list.hasMore)49 #expect(list.rows.first?.id == "run-000" && list.rows.last?.id == "run-039")50 await list.loadMore()51 #expect(list.rows.count == 80 && list.rows.last?.id == "run-079")52 await list.loadMore(); await list.loadMore()53 #expect(list.rows.count == 150 && !list.hasMore)54 #expect(Set(list.rows.map(\.id)).count == 150)55 await list.refresh() // a store change keeps what was loaded56 #expect(list.rows.count == 150)5758 list.filter.search = "invoice"59 await list.reload()60 #expect(list.rows.count == 16) // 15 titles + 1 error summary61 #expect(list.rows.contains { $0.id == "run-007" })6263 list.filter = RunFilter(); list.filter.statuses = [.failed]64 await list.reload()65 #expect(list.rows.count == 30 && list.rows.allSatisfy { $0.status == .failed })6667 list.filter.statuses = [.failed, .timedOut]; list.filter.kinds = ["workflow"]68 await list.reload()69 #expect(list.rows.count == 20 && list.rows.allSatisfy { $0.kind == "workflow" && [.failed, .timedOut].contains($0.status) })70 #expect(!list.hasMore) // a sparse filter is satisfied by scanning further pages7172 list.filter = RunFilter(); list.filter.triggers = [.schedule]73 list.filter.from = Date(timeIntervalSince1970: 1_790_000_000 - 20 * 3600); list.filter.to = Date(timeIntervalSince1970: 1_790_000_000 - 4 * 3600)74 await list.reload()75 #expect(list.rows.map(\.id) == ["run-004", "run-008", "run-012", "run-016", "run-020"])7677 list.filter = RunFilter(); list.filter.newestFirst = false78 await list.reload()79 #expect(list.rows.count == 150 && list.rows.first?.id == "run-149" && list.rows.last?.id == "run-000")80 }8182 @Test func retryIsOfferedOnlyWhenInputsCanBeRecovered() async throws {83 let model = try await PreviewEnvironment.make()84 defer { try? FileManager.default.removeItem(at: model.services.paths.root) }85 let s = model.services86 // An agent run whose objective was recorded as its first message.87 let agent = try #require(try await s.store.run(id: "sample-0"))88 guard case .agent(let request, let note)? = await RetryPlan.recover(for: agent, services: s, defaultWorkspace: model.defaultWorkspace) else {89 Issue.record("an agent run with a recorded objective must be retryable"); return90 }91 #expect(request.objective == "Sample · Organise Downloads" && request.model == agent.model && request.workspaceAccess == .read)92 #expect(note.contains("read access"))93 // No objective anywhere: no retry.94 try await s.store.saveRun(RunRecord(id: "bare", kind: "agent", title: "Draft workflow: x", mode: .review, status: .succeeded))95 #expect(await RetryPlan.recover(for: try #require(try await s.store.run(id: "bare")), services: s, defaultWorkspace: model.defaultWorkspace) == nil)96 // A workflow run whose workflow no longer exists, and a run still in flight: no retry.97 #expect(await RetryPlan.recover(for: try #require(try await s.store.run(id: "sample-1")), services: s, defaultWorkspace: model.defaultWorkspace) == nil)98 try await s.store.saveRun(RunRecord(id: "live", kind: "agent", title: "live", mode: .guided, status: .running))99 #expect(await RetryPlan.recover(for: try #require(try await s.store.run(id: "live")), services: s, defaultWorkspace: model.defaultWorkspace) == nil)100 // A workflow run: retryable without a checkpoint only when the workflow takes no parameters.101 for w in await s.workflows.list() {102 let doc = try await s.workflows.document(workflowID: w.id)103 let record = RunRecord(id: "wf-\(w.id)", kind: "workflow", title: w.name, workflowID: w.id, workflowVersion: w.currentVersion, mode: .scheduled, status: .failed)104 let plan = await RetryPlan.recover(for: record, services: s, defaultWorkspace: model.defaultWorkspace)105 #expect((plan != nil) == doc.spec.parameters.isEmpty, "\(w.id)")106 if case .workflow(let r, _)? = plan { #expect(r.workflowID == w.id && r.version == w.currentVersion && r.mode == .guided && r.trigger == "manual") }107 }108 }109110 @Test func dashboardAndSchedulerLoadRealState() async throws {111 let model = try await PreviewEnvironment.make()112 defer { try? FileManager.default.removeItem(at: model.services.paths.root) }113 let s = model.services114 try await s.store.saveRun(RunRecord(id: "needs-recovery", kind: "agent", title: "Interrupted", mode: .guided, status: .recoveryRequired, startedAt: Date()))115 let dash = DashboardModel()116 await dash.load(s)117 #expect(dash.loaded && dash.days.count == 30 && dash.recent.count == 7)118 #expect(dash.attention.map(\.id) == ["needs-recovery"])119 #expect(dash.workflows.count == 3 && dash.permissions.count == MacPermission.allCases.count)120 let month = PeriodMetrics.aggregate(days: dash.days, runs: dash.periodRuns, period: .month)121 #expect(month.runs == 7 && month.succeeded == 3)122 #expect(month.averageDuration == 42 && month.durationSample == 6)123 #expect(abs(month.costUSD - (0..<6).reduce(0.0) { $0 + 0.004 + Double($1) * 0.003 }) < 1e-9)124125 let workflows = await s.workflows.list()126 // Built-in templates start with their permissions not accepted: enabling one of their schedules must warn.127 #expect(try await s.workflows.permissionReview(workflowID: workflows[0].id).accepted == false)128 var draft = ScheduleDraft()129 draft.workflowID = workflows[0].id; draft.name = workflows[0].name; draft.useRawCron = true; draft.rawCron = "0 9 * * MON"; draft.timezone = "America/Toronto"; draft.enabled = true130 let schedule = draft.makeSchedule(existing: nil)131 try await s.workflows.saveSchedule(schedule)132 let sched = SchedulerModel()133 await sched.load(s)134 let saved = try #require(sched.schedules.first { $0.id == schedule.id })135 #expect(TriggerDescriber.describe(saved.trigger, locale: Locale(identifier: "en_US")) == "Every Monday at 09:00 (America/Toronto)")136 let next = try #require(sched.next[schedule.id])137 #expect(next > Date() && next.timeIntervalSinceNow <= 7 * 86_400 + 60)138 #expect(sched.agenda.count == 7 && sched.agenda.flatMap(\.items).filter { $0.scheduleID == schedule.id }.count == 1)139 #expect(sched.schedulerActive == false)140 try await s.workflows.setSchedule(id: schedule.id, enabled: false)141 await sched.load(s)142 #expect(sched.next[schedule.id] == nil && sched.agenda.flatMap(\.items).allSatisfy { $0.scheduleID != schedule.id })143 try await s.workflows.removeSchedule(id: schedule.id)144 await sched.load(s)145 #expect(!sched.schedules.contains { $0.id == schedule.id })146 }147148 @Test func consoleShowsStoredEventsAndLiveOutputOfARealScript() async throws {149 let model = try await PreviewEnvironment.make(seed: false)150 defer { try? FileManager.default.removeItem(at: model.services.paths.root) }151 let s = model.services152 let runID = "console-run"153 try await s.store.saveRun(RunRecord(id: runID, kind: "script", title: "Console test", mode: .guided, status: .running, startedAt: Date()))154 await s.store.emit(RunEvent(runID: runID, kind: .userMessage, text: "print two lines"))155 let console = ConsoleModel()156 await console.start(runID: runID, services: s)157 #expect(console.isLive)158 #expect(console.visible.map(\.text) == ["print two lines"])159160 let approver = Task { for await pending in s.approvals.updates() { for r in pending { await s.approvals.resolve(id: r.id, response: .approveOnce) } } }161 let hub = s.runs.live162 let call = ToolCall(toolID: "shell.script", arguments: ["script": "echo alpha; echo beta; echo oops >&2; printf 'no newline'"])163 hub.publish(runID, .toolStarted(call))164 let artifacts = s.paths.artifacts.appendingPathComponent(runID)165 try FileManager.default.createDirectory(at: artifacts, withIntermediateDirectories: true)166 let result = await s.executor.invoke(call, context: InvocationContext(runID: runID, mode: .guided, workingDirectory: s.paths.scratch, artifactsDirectory: artifacts,167 progress: { id, p in hub.publish(runID, .toolProgress(callID: id, p)) }))168 approver.cancel()169 #expect(result.status == .succeeded, "\(result.summary)")170 hub.publish(runID, .toolFinished(call, result))171 hub.publish(runID, .finished(.succeeded)); hub.finish(runID)172 try await Task.sleep(nanoseconds: 500_000_000)173 await console.refreshPersisted()174175 let lines = console.buffer.lines176 #expect(lines.contains { $0.level == .stdout && $0.text == "alpha" })177 #expect(lines.contains { $0.level == .stdout && $0.text == "beta" })178 #expect(lines.contains { $0.level == .stdout && $0.text == "no newline" }) // an unfinished last line is not lost179 #expect(lines.contains { $0.level == .stderr && $0.text == "oops" })180 #expect(lines.filter { $0.text.hasPrefix("▶ shell.script") }.count == 1) // live and stored copies of the call collapse181 #expect(lines.filter { $0.text.hasPrefix("■ shell.script succeeded") }.count == 1)182 #expect(lines.contains { $0.level == .approval })183 #expect(lines.last { $0.level == .status }?.text == "SUCCEEDED")184 #expect(!console.isLive)185 #expect(lines.map(\.time) == lines.map(\.time).sorted())186187 console.levels = [.stdout, .stderr]188 #expect(console.visible.count == 4)189 console.query = "a"190 #expect(console.matches.count == 2 && console.currentMatch == 1) // alpha, beta (oops and "no newline" have no a… except "no newline")191 console.clear()192 #expect(console.visible.isEmpty)193 await console.refreshPersisted()194 #expect(console.buffer.lines.isEmpty) // clearing the view does not reload old events195 console.stop()196 }197198 @Test func everyScreenRenders() async throws {199 let model = try await PreviewEnvironment.make()200 defer { try? FileManager.default.removeItem(at: model.services.paths.root) }201 let s = model.services202 let workflows = await s.workflows.list()203 try await s.workflows.saveSchedule(Schedule(workflowID: workflows[0].id, name: workflows[0].name, trigger: .cron(expression: "0 9 * * MON", timezone: "America/Toronto"), enabled: true))204 try await s.store.saveRun(RunRecord(id: "rec", kind: "agent", title: "Interrupted", mode: .guided, status: .recoveryRequired, startedAt: Date()))205 await model.refreshShared()206 #expect(try await render("dashboard", DashboardView(), model, size: CGSize(width: 1280, height: 1500)) > 150_000)207 model.selectedRunID = "sample-0"208 for tab in ["timeline", "receipt", "usage", "console"] {209 #expect(try await render("runs-\(tab)", RunsPreviewScreens.runs(tab: tab), model) > 80_000, "\(tab)")210 }211 for tab in ["schedules", "agenda", "queue"] {212 #expect(try await render("scheduler-\(tab)", RunsPreviewScreens.scheduler(tab: tab), model) > 50_000, "\(tab)")213 }214 let schedule = try #require(await s.workflows.schedules().first { $0.enabled })215 #expect(try await render("editor", RunsPreviewScreens.scheduleEditor(existing: schedule, workflows: workflows), model, size: CGSize(width: 640, height: 720)) > 40_000)216 #expect(try await render("recovery", RunsPreviewScreens.recovery(try #require(try await s.store.run(id: "rec"))), model, size: CGSize(width: 560, height: 420)) > 15_000)217 #expect(try await render("console", ConsoleView(runID: "sample-0").padding(16), model, size: CGSize(width: 1000, height: 500)) > 30_000)218219 let empty = try await PreviewEnvironment.make(seed: false)220 defer { try? FileManager.default.removeItem(at: empty.services.paths.root) }221 #expect(try await render("dashboard-empty", DashboardView(), empty) > 60_000)222 #expect(try await render("runs-empty", RunsView(), empty) > 30_000)223 #expect(try await render("scheduler-empty", SchedulerView(), empty) > 40_000)224 }225}226227@Suite("Performance · console with 50 000 lines", .serialized)228@MainActor229struct ConsolePerformanceTests {230 /// Streams 50 000 synthetic lines in bursts of 500 into a console that is on screen (off-screen window), the way the231 /// 100 ms flush would deliver them, and measures how long the main thread is held per burst, plus search and filter.232 @Test func burstOf50000LinesStaysResponsive() async throws {233 let model = try await PreviewEnvironment.make(seed: false)234 defer { try? FileManager.default.removeItem(at: model.services.paths.root) }235 _ = NSApplication.shared236 let console = ConsoleModel()237 let host = NSHostingView(rootView: RunsPreviewScreens.console(console).environment(model).frame(width: 1100, height: 600))238 host.frame = CGRect(x: 0, y: 0, width: 1100, height: 600)239 let window = NSWindow(contentRect: CGRect(x: -20_000, y: -20_000, width: 1100, height: 600), styleMask: [.borderless], backing: .buffered, defer: false)240 window.contentView = host241 window.orderBack(nil)242 try await Task.sleep(nanoseconds: 300_000_000)243244 let total = 50_000, burst = 500245 let start = Date()246 var held: [Double] = []247 var clock = ContinuousClock.now248 let wall = ContinuousClock.now249 for b in 0..<(total / burst) {250 let entries: [ConsoleAssembler.Entry] = (0..<burst).map { i in251 let n = b * burst + i252 return ("line \(n) of synthetic output: the quick brown fox jumps over the lazy dog \(n % 97 == 0 ? "NEEDLE" : "")", n % 50 == 0 ? .stderr : .stdout,253 start.addingTimeInterval(Double(n) / 1000))254 }255 clock = .now256 console.ingest(entries)257 console.flush()258 host.layoutSubtreeIfNeeded()259 host.displayIfNeeded()260 held.append(Double((ContinuousClock.now - clock).components.attoseconds) / 1e15 + Double((ContinuousClock.now - clock).components.seconds) * 1000)261 await Task.yield() // let SwiftUI apply the change, as between two flush ticks262 if b % 10 == 9 { try await Task.sleep(nanoseconds: 5_000_000) }263 }264 let elapsed = ContinuousClock.now - wall265 let totalMs = Double(elapsed.components.seconds) * 1000 + Double(elapsed.components.attoseconds) / 1e15266267 #expect(console.buffer.lines.count == 20_000)268 #expect(console.dropped == 30_000)269 #expect(console.buffer.lines.first?.text.hasPrefix("line 30000 ") == true)270 #expect(console.buffer.lines.last?.text.hasPrefix("line 49999 ") == true)271272273 func ms(_ work: () -> Void) -> Double {274 let t = ContinuousClock.now; work(); let d = ContinuousClock.now - t275 return Double(d.components.seconds) * 1000 + Double(d.components.attoseconds) / 1e15276 }277 // Search stays correct and cheap while lines keep arriving.278 console.query = "needle"279 let streamingSearchMs = ms {280 console.ingest((0..<500).map { i in ("late line \(i) \(i % 100 == 0 ? "needle" : "")", .stdout, start.addingTimeInterval(60 + Double(i) / 1000)) })281 console.flush()282 }283 #expect(console.matches == console.buffer.search("needle"))284 console.query = ""285 var found = 0286 let searchMs = ms { found = console.buffer.search("needle").count }287 #expect(found == (30_500..<50_000).filter { $0 % 97 == 0 }.count + 5)288 let queryMs = ms { console.query = "needle" }289 let filterMs = ms { console.levels = [.stderr] }290 #expect(console.visible.count == 390)291 let nextMs = ms { for _ in 0..<50 { console.nextMatch() } }292 let copyMs = ms { console.levels = Set(ConsoleLine.Level.allCases); _ = console.text(selection: []) }293 let settle = ms { host.layoutSubtreeIfNeeded(); host.displayIfNeeded() }294 let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("orix-console-50k.png")295 try await ShotRenderer.render(RunsPreviewScreens.console(console).environment(model), size: CGSize(width: 1100, height: 600), settle: 0.6, to: url)296 #expect(pngSize(url) > 40_000)297 try? FileManager.default.removeItem(at: url)298 window.orderOut(nil)299300 let sorted = held.sorted()301 let mean = held.reduce(0, +) / Double(held.count), p95 = sorted[Int(Double(sorted.count) * 0.95)], worst = sorted.last ?? 0302 print(String(format: "MEASURE console 50k: %d bursts of %d lines, total %.0f ms; main thread held per burst (ingest + flush + layout + draw): mean %.1f ms, p95 %.1f ms, max %.1f ms; search over 20 000 lines %.1f ms; set query (search + refresh) %.1f ms; level filter %.1f ms; 50 next-match steps %.2f ms; copy all 20 000 lines %.1f ms; relayout after filter %.1f ms; one 500-line flush with a search active %.1f ms",303 held.count, burst, totalMs, mean, p95, worst, searchMs, queryMs, filterMs, nextMs, copyMs, settle, streamingSearchMs))304 // Budgets: a burst must fit well inside the 100 ms flush period; interactions must feel immediate.305 #expect(mean < 50)306 #expect(p95 < 100)307 #expect(searchMs < 500)308 #expect(queryMs < 1000)309 #expect(filterMs < 500)310 #expect(streamingSearchMs < 100)311 }312}313