SPB Git

Repository Public License

IntegrationTests.swift 19.8 KB 313 lines swift

1// ORIX OS Automation2// File: Packages/ORIXFeatureRuns/Tests/ORIXFeatureRunsTests/IntegrationTests.swift3// Description: Integration tests on real services in a temporary folder (real SQLite, real subprocess): run list paging and filters, retry recovery, dashboard and scheduler loading, the console fed by the store and the live hub, one render per screen, and the 50 000-line console measurement.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import AppKit8import Foundation9import ORIXAppUI10@_spi(Preview) @testable import ORIXFeatureRuns11import SwiftUI12import Testing1314@MainActor15private func seedMany(_ store: ORIXStore, count: Int) async throws {16    let base = Date(timeIntervalSince1970: 1_790_000_000)17    let statuses: [RunStatus] = [.succeeded, .failed, .canceled, .succeeded, .timedOut]18    for i in 0..<count {19        let start = base.addingTimeInterval(Double(-i) * 3600)   // run 0 is the newest20        var r = RunRecord(id: String(format: "run-%03d", i), kind: i % 3 == 0 ? "workflow" : "agent", title: i % 10 == 0 ? "Invoice batch \(i)" : "Routine \(i)",21                          trigger: i % 4 == 0 ? "cron" : "manual", mode: .guided, status: statuses[i % statuses.count], queuedAt: start, startedAt: start,22                          endedAt: start.addingTimeInterval(30))23        if i == 7 { r.errorSummary = "the invoice folder is missing" }24        try await store.saveRun(r)25    }26}2728private func pngSize(_ url: URL) -> Int { (try? FileManager.default.attributesOfItem(atPath: url.path)[.size] as? Int) ?? 0 }2930@MainActor31private func render<V: View>(_ name: String, _ view: V, _ model: AppModel, size: CGSize = CGSize(width: 1280, height: 820)) async throws -> Int {32    let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("orix-runs-test-\(name).png")33    try await ShotRenderer.render(view.environment(model), size: size, to: url)34    defer { try? FileManager.default.removeItem(at: url) }35    return pngSize(url)36}3738@Suite("Integration · real store and services", .serialized)39@MainActor40struct RunsIntegrationTests {41    @Test func listPagesSearchesAndFilters() async throws {42        let model = try await PreviewEnvironment.make(seed: false)43        defer { try? FileManager.default.removeItem(at: model.services.paths.root) }44        try await seedMany(model.services.store, count: 150)45        let list = RunsListModel(store: model.services.store, pageSize: 40)4647        await list.reload()48        #expect(list.rows.count == 40 && list.hasMore)49        #expect(list.rows.first?.id == "run-000" && list.rows.last?.id == "run-039")50        await list.loadMore()51        #expect(list.rows.count == 80 && list.rows.last?.id == "run-079")52        await list.loadMore(); await list.loadMore()53        #expect(list.rows.count == 150 && !list.hasMore)54        #expect(Set(list.rows.map(\.id)).count == 150)55        await list.refresh()                       // a store change keeps what was loaded56        #expect(list.rows.count == 150)5758        list.filter.search = "invoice"59        await list.reload()60        #expect(list.rows.count == 16)             // 15 titles + 1 error summary61        #expect(list.rows.contains { $0.id == "run-007" })6263        list.filter = RunFilter(); list.filter.statuses = [.failed]64        await list.reload()65        #expect(list.rows.count == 30 && list.rows.allSatisfy { $0.status == .failed })6667        list.filter.statuses = [.failed, .timedOut]; list.filter.kinds = ["workflow"]68        await list.reload()69        #expect(list.rows.count == 20 && list.rows.allSatisfy { $0.kind == "workflow" && [.failed, .timedOut].contains($0.status) })70        #expect(!list.hasMore)                     // a sparse filter is satisfied by scanning further pages7172        list.filter = RunFilter(); list.filter.triggers = [.schedule]73        list.filter.from = Date(timeIntervalSince1970: 1_790_000_000 - 20 * 3600); list.filter.to = Date(timeIntervalSince1970: 1_790_000_000 - 4 * 3600)74        await list.reload()75        #expect(list.rows.map(\.id) == ["run-004", "run-008", "run-012", "run-016", "run-020"])7677        list.filter = RunFilter(); list.filter.newestFirst = false78        await list.reload()79        #expect(list.rows.count == 150 && list.rows.first?.id == "run-149" && list.rows.last?.id == "run-000")80    }8182    @Test func retryIsOfferedOnlyWhenInputsCanBeRecovered() async throws {83        let model = try await PreviewEnvironment.make()84        defer { try? FileManager.default.removeItem(at: model.services.paths.root) }85        let s = model.services86        // An agent run whose objective was recorded as its first message.87        let agent = try #require(try await s.store.run(id: "sample-0"))88        guard case .agent(let request, let note)? = await RetryPlan.recover(for: agent, services: s, defaultWorkspace: model.defaultWorkspace) else {89            Issue.record("an agent run with a recorded objective must be retryable"); return90        }91        #expect(request.objective == "Sample · Organise Downloads" && request.model == agent.model && request.workspaceAccess == .read)92        #expect(note.contains("read access"))93        // No objective anywhere: no retry.94        try await s.store.saveRun(RunRecord(id: "bare", kind: "agent", title: "Draft workflow: x", mode: .review, status: .succeeded))95        #expect(await RetryPlan.recover(for: try #require(try await s.store.run(id: "bare")), services: s, defaultWorkspace: model.defaultWorkspace) == nil)96        // A workflow run whose workflow no longer exists, and a run still in flight: no retry.97        #expect(await RetryPlan.recover(for: try #require(try await s.store.run(id: "sample-1")), services: s, defaultWorkspace: model.defaultWorkspace) == nil)98        try await s.store.saveRun(RunRecord(id: "live", kind: "agent", title: "live", mode: .guided, status: .running))99        #expect(await RetryPlan.recover(for: try #require(try await s.store.run(id: "live")), services: s, defaultWorkspace: model.defaultWorkspace) == nil)100        // A workflow run: retryable without a checkpoint only when the workflow takes no parameters.101        for w in await s.workflows.list() {102            let doc = try await s.workflows.document(workflowID: w.id)103            let record = RunRecord(id: "wf-\(w.id)", kind: "workflow", title: w.name, workflowID: w.id, workflowVersion: w.currentVersion, mode: .scheduled, status: .failed)104            let plan = await RetryPlan.recover(for: record, services: s, defaultWorkspace: model.defaultWorkspace)105            #expect((plan != nil) == doc.spec.parameters.isEmpty, "\(w.id)")106            if case .workflow(let r, _)? = plan { #expect(r.workflowID == w.id && r.version == w.currentVersion && r.mode == .guided && r.trigger == "manual") }107        }108    }109110    @Test func dashboardAndSchedulerLoadRealState() async throws {111        let model = try await PreviewEnvironment.make()112        defer { try? FileManager.default.removeItem(at: model.services.paths.root) }113        let s = model.services114        try await s.store.saveRun(RunRecord(id: "needs-recovery", kind: "agent", title: "Interrupted", mode: .guided, status: .recoveryRequired, startedAt: Date()))115        let dash = DashboardModel()116        await dash.load(s)117        #expect(dash.loaded && dash.days.count == 30 && dash.recent.count == 7)118        #expect(dash.attention.map(\.id) == ["needs-recovery"])119        #expect(dash.workflows.count == 3 && dash.permissions.count == MacPermission.allCases.count)120        let month = PeriodMetrics.aggregate(days: dash.days, runs: dash.periodRuns, period: .month)121        #expect(month.runs == 7 && month.succeeded == 3)122        #expect(month.averageDuration == 42 && month.durationSample == 6)123        #expect(abs(month.costUSD - (0..<6).reduce(0.0) { $0 + 0.004 + Double($1) * 0.003 }) < 1e-9)124125        let workflows = await s.workflows.list()126        // Built-in templates start with their permissions not accepted: enabling one of their schedules must warn.127        #expect(try await s.workflows.permissionReview(workflowID: workflows[0].id).accepted == false)128        var draft = ScheduleDraft()129        draft.workflowID = workflows[0].id; draft.name = workflows[0].name; draft.useRawCron = true; draft.rawCron = "0 9 * * MON"; draft.timezone = "America/Toronto"; draft.enabled = true130        let schedule = draft.makeSchedule(existing: nil)131        try await s.workflows.saveSchedule(schedule)132        let sched = SchedulerModel()133        await sched.load(s)134        let saved = try #require(sched.schedules.first { $0.id == schedule.id })135        #expect(TriggerDescriber.describe(saved.trigger, locale: Locale(identifier: "en_US")) == "Every Monday at 09:00 (America/Toronto)")136        let next = try #require(sched.next[schedule.id])137        #expect(next > Date() && next.timeIntervalSinceNow <= 7 * 86_400 + 60)138        #expect(sched.agenda.count == 7 && sched.agenda.flatMap(\.items).filter { $0.scheduleID == schedule.id }.count == 1)139        #expect(sched.schedulerActive == false)140        try await s.workflows.setSchedule(id: schedule.id, enabled: false)141        await sched.load(s)142        #expect(sched.next[schedule.id] == nil && sched.agenda.flatMap(\.items).allSatisfy { $0.scheduleID != schedule.id })143        try await s.workflows.removeSchedule(id: schedule.id)144        await sched.load(s)145        #expect(!sched.schedules.contains { $0.id == schedule.id })146    }147148    @Test func consoleShowsStoredEventsAndLiveOutputOfARealScript() async throws {149        let model = try await PreviewEnvironment.make(seed: false)150        defer { try? FileManager.default.removeItem(at: model.services.paths.root) }151        let s = model.services152        let runID = "console-run"153        try await s.store.saveRun(RunRecord(id: runID, kind: "script", title: "Console test", mode: .guided, status: .running, startedAt: Date()))154        await s.store.emit(RunEvent(runID: runID, kind: .userMessage, text: "print two lines"))155        let console = ConsoleModel()156        await console.start(runID: runID, services: s)157        #expect(console.isLive)158        #expect(console.visible.map(\.text) == ["print two lines"])159160        let approver = Task { for await pending in s.approvals.updates() { for r in pending { await s.approvals.resolve(id: r.id, response: .approveOnce) } } }161        let hub = s.runs.live162        let call = ToolCall(toolID: "shell.script", arguments: ["script": "echo alpha; echo beta; echo oops >&2; printf 'no newline'"])163        hub.publish(runID, .toolStarted(call))164        let artifacts = s.paths.artifacts.appendingPathComponent(runID)165        try FileManager.default.createDirectory(at: artifacts, withIntermediateDirectories: true)166        let result = await s.executor.invoke(call, context: InvocationContext(runID: runID, mode: .guided, workingDirectory: s.paths.scratch, artifactsDirectory: artifacts,167                                                                              progress: { id, p in hub.publish(runID, .toolProgress(callID: id, p)) }))168        approver.cancel()169        #expect(result.status == .succeeded, "\(result.summary)")170        hub.publish(runID, .toolFinished(call, result))171        hub.publish(runID, .finished(.succeeded)); hub.finish(runID)172        try await Task.sleep(nanoseconds: 500_000_000)173        await console.refreshPersisted()174175        let lines = console.buffer.lines176        #expect(lines.contains { $0.level == .stdout && $0.text == "alpha" })177        #expect(lines.contains { $0.level == .stdout && $0.text == "beta" })178        #expect(lines.contains { $0.level == .stdout && $0.text == "no newline" })     // an unfinished last line is not lost179        #expect(lines.contains { $0.level == .stderr && $0.text == "oops" })180        #expect(lines.filter { $0.text.hasPrefix("▶ shell.script") }.count == 1)        // live and stored copies of the call collapse181        #expect(lines.filter { $0.text.hasPrefix("■ shell.script succeeded") }.count == 1)182        #expect(lines.contains { $0.level == .approval })183        #expect(lines.last { $0.level == .status }?.text == "SUCCEEDED")184        #expect(!console.isLive)185        #expect(lines.map(\.time) == lines.map(\.time).sorted())186187        console.levels = [.stdout, .stderr]188        #expect(console.visible.count == 4)189        console.query = "a"190        #expect(console.matches.count == 2 && console.currentMatch == 1)               // alpha, beta (oops and "no newline" have no a… except "no newline")191        console.clear()192        #expect(console.visible.isEmpty)193        await console.refreshPersisted()194        #expect(console.buffer.lines.isEmpty)                                          // clearing the view does not reload old events195        console.stop()196    }197198    @Test func everyScreenRenders() async throws {199        let model = try await PreviewEnvironment.make()200        defer { try? FileManager.default.removeItem(at: model.services.paths.root) }201        let s = model.services202        let workflows = await s.workflows.list()203        try await s.workflows.saveSchedule(Schedule(workflowID: workflows[0].id, name: workflows[0].name, trigger: .cron(expression: "0 9 * * MON", timezone: "America/Toronto"), enabled: true))204        try await s.store.saveRun(RunRecord(id: "rec", kind: "agent", title: "Interrupted", mode: .guided, status: .recoveryRequired, startedAt: Date()))205        await model.refreshShared()206        #expect(try await render("dashboard", DashboardView(), model, size: CGSize(width: 1280, height: 1500)) > 150_000)207        model.selectedRunID = "sample-0"208        for tab in ["timeline", "receipt", "usage", "console"] {209            #expect(try await render("runs-\(tab)", RunsPreviewScreens.runs(tab: tab), model) > 80_000, "\(tab)")210        }211        for tab in ["schedules", "agenda", "queue"] {212            #expect(try await render("scheduler-\(tab)", RunsPreviewScreens.scheduler(tab: tab), model) > 50_000, "\(tab)")213        }214        let schedule = try #require(await s.workflows.schedules().first { $0.enabled })215        #expect(try await render("editor", RunsPreviewScreens.scheduleEditor(existing: schedule, workflows: workflows), model, size: CGSize(width: 640, height: 720)) > 40_000)216        #expect(try await render("recovery", RunsPreviewScreens.recovery(try #require(try await s.store.run(id: "rec"))), model, size: CGSize(width: 560, height: 420)) > 15_000)217        #expect(try await render("console", ConsoleView(runID: "sample-0").padding(16), model, size: CGSize(width: 1000, height: 500)) > 30_000)218219        let empty = try await PreviewEnvironment.make(seed: false)220        defer { try? FileManager.default.removeItem(at: empty.services.paths.root) }221        #expect(try await render("dashboard-empty", DashboardView(), empty) > 60_000)222        #expect(try await render("runs-empty", RunsView(), empty) > 30_000)223        #expect(try await render("scheduler-empty", SchedulerView(), empty) > 40_000)224    }225}226227@Suite("Performance · console with 50 000 lines", .serialized)228@MainActor229struct ConsolePerformanceTests {230    /// Streams 50 000 synthetic lines in bursts of 500 into a console that is on screen (off-screen window), the way the231    /// 100 ms flush would deliver them, and measures how long the main thread is held per burst, plus search and filter.232    @Test func burstOf50000LinesStaysResponsive() async throws {233        let model = try await PreviewEnvironment.make(seed: false)234        defer { try? FileManager.default.removeItem(at: model.services.paths.root) }235        _ = NSApplication.shared236        let console = ConsoleModel()237        let host = NSHostingView(rootView: RunsPreviewScreens.console(console).environment(model).frame(width: 1100, height: 600))238        host.frame = CGRect(x: 0, y: 0, width: 1100, height: 600)239        let window = NSWindow(contentRect: CGRect(x: -20_000, y: -20_000, width: 1100, height: 600), styleMask: [.borderless], backing: .buffered, defer: false)240        window.contentView = host241        window.orderBack(nil)242        try await Task.sleep(nanoseconds: 300_000_000)243244        let total = 50_000, burst = 500245        let start = Date()246        var held: [Double] = []247        var clock = ContinuousClock.now248        let wall = ContinuousClock.now249        for b in 0..<(total / burst) {250            let entries: [ConsoleAssembler.Entry] = (0..<burst).map { i in251                let n = b * burst + i252                return ("line \(n) of synthetic output: the quick brown fox jumps over the lazy dog \(n % 97 == 0 ? "NEEDLE" : "")", n % 50 == 0 ? .stderr : .stdout,253                        start.addingTimeInterval(Double(n) / 1000))254            }255            clock = .now256            console.ingest(entries)257            console.flush()258            host.layoutSubtreeIfNeeded()259            host.displayIfNeeded()260            held.append(Double((ContinuousClock.now - clock).components.attoseconds) / 1e15 + Double((ContinuousClock.now - clock).components.seconds) * 1000)261            await Task.yield()   // let SwiftUI apply the change, as between two flush ticks262            if b % 10 == 9 { try await Task.sleep(nanoseconds: 5_000_000) }263        }264        let elapsed = ContinuousClock.now - wall265        let totalMs = Double(elapsed.components.seconds) * 1000 + Double(elapsed.components.attoseconds) / 1e15266267        #expect(console.buffer.lines.count == 20_000)268        #expect(console.dropped == 30_000)269        #expect(console.buffer.lines.first?.text.hasPrefix("line 30000 ") == true)270        #expect(console.buffer.lines.last?.text.hasPrefix("line 49999 ") == true)271272273        func ms(_ work: () -> Void) -> Double {274            let t = ContinuousClock.now; work(); let d = ContinuousClock.now - t275            return Double(d.components.seconds) * 1000 + Double(d.components.attoseconds) / 1e15276        }277        // Search stays correct and cheap while lines keep arriving.278        console.query = "needle"279        let streamingSearchMs = ms {280            console.ingest((0..<500).map { i in ("late line \(i) \(i % 100 == 0 ? "needle" : "")", .stdout, start.addingTimeInterval(60 + Double(i) / 1000)) })281            console.flush()282        }283        #expect(console.matches == console.buffer.search("needle"))284        console.query = ""285        var found = 0286        let searchMs = ms { found = console.buffer.search("needle").count }287        #expect(found == (30_500..<50_000).filter { $0 % 97 == 0 }.count + 5)288        let queryMs = ms { console.query = "needle" }289        let filterMs = ms { console.levels = [.stderr] }290        #expect(console.visible.count == 390)291        let nextMs = ms { for _ in 0..<50 { console.nextMatch() } }292        let copyMs = ms { console.levels = Set(ConsoleLine.Level.allCases); _ = console.text(selection: []) }293        let settle = ms { host.layoutSubtreeIfNeeded(); host.displayIfNeeded() }294        let url = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("orix-console-50k.png")295        try await ShotRenderer.render(RunsPreviewScreens.console(console).environment(model), size: CGSize(width: 1100, height: 600), settle: 0.6, to: url)296        #expect(pngSize(url) > 40_000)297        try? FileManager.default.removeItem(at: url)298        window.orderOut(nil)299300        let sorted = held.sorted()301        let mean = held.reduce(0, +) / Double(held.count), p95 = sorted[Int(Double(sorted.count) * 0.95)], worst = sorted.last ?? 0302        print(String(format: "MEASURE console 50k: %d bursts of %d lines, total %.0f ms; main thread held per burst (ingest + flush + layout + draw): mean %.1f ms, p95 %.1f ms, max %.1f ms; search over 20 000 lines %.1f ms; set query (search + refresh) %.1f ms; level filter %.1f ms; 50 next-match steps %.2f ms; copy all 20 000 lines %.1f ms; relayout after filter %.1f ms; one 500-line flush with a search active %.1f ms",303                     held.count, burst, totalMs, mean, p95, worst, searchMs, queryMs, filterMs, nextMs, copyMs, settle, streamingSearchMs))304        // Budgets: a burst must fit well inside the 100 ms flush period; interactions must feel immediate.305        #expect(mean < 50)306        #expect(p95 < 100)307        #expect(searchMs < 500)308        #expect(queryMs < 1000)309        #expect(filterMs < 500)310        #expect(streamingSearchMs < 100)311    }312}313
Files main
Menu

Theme