SPB Git

Repository Public License

WorkflowTests.swift 37.3 KB 715 lines swift

1// ORIX Secure2// File: Packages/ORIXReporting/Tests/ORIXWorkflowTests/WorkflowTests.swift3// Description: Tests of the workflow engine: validation, event order, progress, confirmation gates, cancellation, failures,4//              time limits, context passing, run reports, the built-in library, persistence and parallel runs.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import ORIXSecureModel10import Testing11@testable import ORIXWorkflow1213// MARK: - Helpers1415/// Thread-safe log of what the handlers and the confirmation closure were asked to do.16final class CallLog: @unchecked Sendable {17    private let lock = NSLock()18    private var calls: [String] = []19    func add(_ call: String) { lock.withLock { calls.append(call) } }20    var all: [String] { lock.withLock { calls } }21    func count(_ call: String) -> Int { lock.withLock { calls.filter { $0 == call }.count } }22}2324/// A one-shot signal a test can wait for.25final class Signal: Sendable {26    private let stream: AsyncStream<Void>27    private let continuation: AsyncStream<Void>.Continuation28    init() { (stream, continuation) = AsyncStream.makeStream(of: Void.self) }29    func fire() { continuation.yield() }30    func wait() async { for await _ in stream { return } }31}3233struct TestFailure: Error, LocalizedError {34    var errorDescription: String? { "The disk could not be measured." }35}3637func registry(_ log: CallLog, rules: WorkflowRules = .standard, kinds: [WorkflowStepKind]? = nil) -> WorkflowHandlerRegistry {38    var registry = WorkflowHandlerRegistry(rules: rules)39    let all: [WorkflowStepKind] = [.securityAssessment, .storageOverview, .photosScan, .photosReview, .photosDelete, .networkDiagnostics, .urlInspect, .checklistReview, .reportGenerate]40    for kind in kinds ?? all {41        registry.register(kind) { step, _, _ in42            log.add(kind.rawValue)43            return StepOutput(summary: "done \(step.kind.rawValue)")44        }45    }46    return registry47}4849func label(_ event: WorkflowEvent) -> String {50    switch event {51    case .started: "started"52    case .stepStarted(let index, _): "stepStarted(\(index))"53    case .stepProgress(let index, let fraction, let message): "stepProgress(\(index),\(fraction),\(message ?? "-"))"54    case .awaitingConfirmation(let index, _): "awaitingConfirmation(\(index))"55    case .stepFinished(let index, _, _): "stepFinished(\(index))"56    case .stepSkipped(let index, _, _): "stepSkipped(\(index))"57    case .stepFailed(let index, _, _): "stepFailed(\(index))"58    case .finished(let run): "finished(\(run.status.rawValue))"59    }60}6162/// Runs to the end and returns the event labels and the final run.63func collect(_ runner: WorkflowRunner, _ definition: WorkflowDefinition, _ handlers: WorkflowHandlerRegistry, initialValues: [String: String] = [:], confirm: @escaping @Sendable (WorkflowStep) async -> ConfirmationDecision = { _ in .confirmed }) async throws -> (events: [String], run: WorkflowRun) {64    var labels: [String] = []65    var final: WorkflowRun?66    let stream = await runner.run(definition, handlers: handlers, initialValues: initialValues, confirm: confirm)67    for await event in stream {68        labels.append(label(event))69        if case .finished(let run) = event { final = run }70    }71    let run = try #require(final)72    return (labels, run)73}7475func statuses(_ run: WorkflowRun) -> [WorkflowStepStatus] { run.steps.map(\.status) }7677func step(_ kind: WorkflowStepKind, confirm: Bool = false, destructive: Bool = false, continueOnFailure: Bool = false, stopWhenDeclined: Bool = true, timeout: TimeInterval? = nil, parameters: [String: String] = [:]) -> WorkflowStep {78    WorkflowStep(kind: kind, title: "Step \(kind.rawValue)", parameters: parameters, requiresConfirmation: confirm, isDestructive: destructive, continueOnFailure: continueOnFailure, stopsWorkflowWhenDeclined: stopWhenDeclined, timeout: timeout)79}8081func cleanup(stopWhenDeclined: Bool = true) throws -> WorkflowDefinition {82    try WorkflowDefinition(name: "Cleanup", summary: "Test cleanup", steps: [83        step(.photosScan),84        step(.photosReview),85        step(.photosDelete, confirm: true, destructive: true, stopWhenDeclined: stopWhenDeclined),86        step(.reportGenerate),87    ])88}8990// MARK: - Validation9192@Suite("Workflow validation")93struct WorkflowValidationTests {94    private func expect(_ expected: WorkflowValidationError, name: String = "W", rules: WorkflowRules = .standard, _ steps: [WorkflowStep]) {95        #expect(throws: expected) { try WorkflowDefinition(name: name, summary: "", steps: steps, rules: rules) }96    }9798    @Test func destructiveStepMustAskForConfirmation() {99        let delete = step(.photosDelete, confirm: false, destructive: true)100        expect(.destructiveWithoutConfirmation(stepTitle: delete.title), [step(.photosReview), delete])101        #expect(throws: WorkflowValidationError.destructiveWithoutConfirmation(stepTitle: delete.title)) { try delete.validate() }102    }103104    @Test func destructiveKindMustBeFlagged() {105        let sneaky = step(.photosDelete, confirm: false, destructive: false)106        expect(.destructiveKindNotFlagged(stepTitle: sneaky.title, kind: .photosDelete), [step(.photosReview), sneaky])107    }108109    @Test func destructiveStepNeedsItsReviewFirst() {110        let delete = step(.photosDelete, confirm: true, destructive: true)111        expect(.missingReviewPredecessor(stepTitle: delete.title, required: .photosReview), [step(.photosScan), delete])112        // A review placed after the deletion does not count.113        expect(.missingReviewPredecessor(stepTitle: delete.title, required: .photosReview), [delete, step(.photosReview)])114        // A destructive step of a kind for which no review is declared is refused.115        let custom = step("files.delete", confirm: true, destructive: true)116        expect(.destructiveWithoutDeclaredReview(stepTitle: custom.title, kind: "files.delete"), [step(.photosReview), custom])117        // Declared in the rules, it is accepted.118        let rules = WorkflowRules(destructiveKinds: ["files.delete"], requiredPredecessors: ["files.delete": "files.review"])119        #expect(throws: Never.self) { try WorkflowDefinition(name: "Files", summary: "", steps: [step("files.review"), custom], rules: rules) }120    }121122    @Test func namesStepCountAndIdentifiers() {123        expect(.emptyName, name: "  \n", [step(.photosScan)])124        expect(.noSteps, [])125        expect(.tooManySteps(count: 25, maximum: 24), (0..<25).map { _ in step(.photosScan) })126        expect(.tooManySteps(count: 2, maximum: 1), rules: WorkflowRules(maximumStepCount: 1), [step(.photosScan), step(.photosScan)])127        let one = step(.photosScan)128        expect(.duplicateStepIdentifier(stepTitle: one.title), [one, one])129        expect(.emptyStepTitle(kind: .photosScan), [WorkflowStep(kind: .photosScan, title: " ")])130        expect(.emptyStepKind(stepTitle: "T"), [WorkflowStep(kind: "", title: "T")])131        expect(.invalidTimeout(stepTitle: one.title), [step(.photosScan, timeout: 0)])132        #expect(WorkflowValidationError.allCasesHaveText)133    }134135    @Test func decodingAnInvalidDefinitionFails() throws {136        let valid = try cleanup()137        let data = try JSONEncoder().encode(valid)138        let decoded = try JSONDecoder().decode(WorkflowDefinition.self, from: data)139        #expect(decoded == valid)140141        // The same file with the confirmation of the destructive step switched off.142        let object = try JSONSerialization.jsonObject(with: data)143        var root = try #require(object as? [String: Any])144        var steps = try #require(root["steps"] as? [[String: Any]])145        #expect(steps[2]["isDestructive"] as? Bool == true)146        steps[2]["requiresConfirmation"] = false147        root["steps"] = steps148        let tampered = try JSONSerialization.data(withJSONObject: root)149        #expect(throws: DecodingError.self) { try JSONDecoder().decode(WorkflowDefinition.self, from: tampered) }150151        // The destructive flag removed: the kind is still known to be destructive.152        steps[2]["requiresConfirmation"] = true153        steps[2]["isDestructive"] = false154        root["steps"] = steps155        let unflagged = try JSONSerialization.data(withJSONObject: root)156        #expect(throws: DecodingError.self) { try JSONDecoder().decode(WorkflowDefinition.self, from: unflagged) }157158        // Safety flags missing altogether: rejected, not defaulted.159        steps[2]["isDestructive"] = nil160        steps[2]["requiresConfirmation"] = nil161        root["steps"] = steps162        let missing = try JSONSerialization.data(withJSONObject: root)163        #expect(throws: DecodingError.self) { try JSONDecoder().decode(WorkflowDefinition.self, from: missing) }164165        // The review step removed.166        steps = try #require((try JSONSerialization.jsonObject(with: data) as? [String: Any])?["steps"] as? [[String: Any]])167        steps.remove(at: 1)168        root["steps"] = steps169        let unreviewed = try JSONSerialization.data(withJSONObject: root)170        #expect(throws: DecodingError.self) { try JSONDecoder().decode(WorkflowDefinition.self, from: unreviewed) }171172        // Rules supplied through the decoder are honoured.173        let strict = JSONDecoder()174        let key = try #require(CodingUserInfoKey.workflowRules)175        strict.userInfo[key] = WorkflowRules(destructiveKinds: [.photosDelete], requiredPredecessors: [.photosDelete: .photosReview], maximumStepCount: 2)176        #expect(throws: DecodingError.self) { try strict.decode(WorkflowDefinition.self, from: data) }177    }178179    @Test func updatingAndDuplicatingRevalidate() throws {180        let definition = try cleanup()181        let renamed = try definition.updating(name: "Renamed")182        #expect(renamed.id == definition.id && renamed.name == "Renamed" && renamed.steps == definition.steps)183        #expect(throws: WorkflowValidationError.self) { try definition.updating(steps: [definition.steps[2]]) }184        let copy = try WorkflowLibrary.storageCleanup.duplicated(name: "Mine")185        #expect(!copy.isBuiltIn && copy.id != WorkflowLibrary.storageCleanup.id)186        #expect(copy.steps.map(\.kind) == WorkflowLibrary.storageCleanup.steps.map(\.kind))187        #expect(Set(copy.steps.map(\.id)).isDisjoint(with: WorkflowLibrary.storageCleanup.steps.map(\.id)))188    }189}190191extension WorkflowValidationError {192    static var allCasesHaveText: Bool {193        let cases: [WorkflowValidationError] = [194            .emptyName, .noSteps, .tooManySteps(count: 2, maximum: 1), .duplicateStepIdentifier(stepTitle: "a"), .emptyStepTitle(kind: "k"),195            .emptyStepKind(stepTitle: "a"), .invalidTimeout(stepTitle: "a"), .destructiveWithoutConfirmation(stepTitle: "a"),196            .destructiveKindNotFlagged(stepTitle: "a", kind: "k"), .destructiveWithoutDeclaredReview(stepTitle: "a", kind: "k"),197            .missingReviewPredecessor(stepTitle: "a", required: "k"),198        ]199        return cases.allSatisfy { !($0.errorDescription ?? "").isEmpty }200    }201}202203// MARK: - Running204205@Suite("Workflow runner")206struct WorkflowRunnerTests {207    @Test func happyPathEventOrder() async throws {208        let log = CallLog()209        let definition = try cleanup()210        let runner = WorkflowRunner()211        let result = try await collect(runner, definition, registry(log)) { step in212            log.add("confirm:\(step.kind.rawValue)")213            return .confirmed214        }215        #expect(result.events == [216            "started",217            "stepStarted(0)", "stepFinished(0)",218            "stepStarted(1)", "stepFinished(1)",219            "stepStarted(2)", "awaitingConfirmation(2)", "stepFinished(2)",220            "stepStarted(3)", "stepFinished(3)",221            "finished(succeeded)",222        ])223        #expect(log.all == ["photos.scan", "photos.review", "confirm:photos.delete", "photos.delete", "report.generate"])224        let run = result.run225        #expect(run.status == .succeeded)226        #expect(statuses(run) == [.succeeded, .succeeded, .succeeded, .succeeded])227        #expect(run.steps.map(\.summary) == ["done photos.scan", "done photos.review", "done photos.delete", "done report.generate"])228        #expect(run.steps.allSatisfy { $0.duration != nil && $0.startedAt != nil && $0.errorText == nil })229        #expect(run.definition == definition)230        #expect(run.endedAt >= run.startedAt)231        let history = await runner.history232        #expect(history == [run])233        let running = await runner.isRunning234        #expect(!running)235    }236237    @Test func progressIsForwardedAndClamped() async throws {238        var handlers = WorkflowHandlerRegistry()239        handlers.register(.photosScan) { _, _, progress in240            progress(0, "starting")241            progress(0.5, nil)242            progress(7, "over")243            progress(-.infinity, "bad")244            return StepOutput(summary: "ok")245        }246        let definition = try WorkflowDefinition(name: "P", summary: "", steps: [step(.photosScan)])247        let result = try await collect(WorkflowRunner(), definition, handlers)248        #expect(result.events == [249            "started", "stepStarted(0)",250            "stepProgress(0,0.0,starting)", "stepProgress(0,0.5,-)", "stepProgress(0,1.0,over)", "stepProgress(0,0.0,bad)",251            "stepFinished(0)", "finished(succeeded)",252        ])253    }254255    @Test func declinedConfirmationNeverRunsTheDestructiveStep() async throws {256        // Default: declining stops the workflow.257        let log = CallLog()258        let stopping = try cleanup(stopWhenDeclined: true)259        let stopped = try await collect(WorkflowRunner(), stopping, registry(log)) { _ in .declined }260        #expect(log.count("photos.delete") == 0)261        #expect(log.count("report.generate") == 0)262        #expect(stopped.run.status == .declined)263        #expect(statuses(stopped.run) == [.succeeded, .succeeded, .declined, .notRun])264        #expect(stopped.events.suffix(4) == ["stepStarted(2)", "awaitingConfirmation(2)", "stepSkipped(2)", "finished(declined)"])265        #expect(stopped.run.steps[3].startedAt == nil && stopped.run.steps[3].duration == nil)266267        // Per-step option: declining skips the step and the workflow goes on.268        let log2 = CallLog()269        let continuing = try cleanup(stopWhenDeclined: false)270        let continued = try await collect(WorkflowRunner(), continuing, registry(log2)) { _ in .declined }271        #expect(log2.count("photos.delete") == 0)272        #expect(log2.count("report.generate") == 1)273        #expect(continued.run.status == .completedWithIssues)274        #expect(statuses(continued.run) == [.succeeded, .succeeded, .declined, .succeeded])275276        // Many runs, always declined: the destructive handler is never reached.277        let log3 = CallLog()278        for _ in 0..<25 {279            _ = try await collect(WorkflowRunner(), continuing, registry(log3)) { _ in .declined }280        }281        #expect(log3.count("photos.delete") == 0)282        #expect(log3.count("photos.review") == 25)283    }284285    @Test func destructiveStepIsSkippedWhenItsReviewDidNotSucceed() async throws {286        let log = CallLog()287        var handlers = registry(log)288        handlers.register(.photosReview) { _, _, _ in throw TestFailure() }289        var steps = try cleanup().steps290        steps[1].continueOnFailure = true291        let definition = try WorkflowDefinition(name: "Cleanup", summary: "", steps: steps)292        let result = try await collect(WorkflowRunner(), definition, handlers) { step in293            log.add("confirm:\(step.kind.rawValue)")294            return .confirmed295        }296        #expect(log.count("photos.delete") == 0)297        #expect(log.count("confirm:photos.delete") == 0) // the user is not even asked298        #expect(statuses(result.run) == [.succeeded, .failed, .skipped, .succeeded])299        #expect(result.events.contains("stepSkipped(2)"))300        #expect(result.run.status == .completedWithIssues)301        #expect(result.run.steps[2].summary?.contains("photos.review") == true)302    }303304    @Test func cancellationDuringAStepStopsAndNeverDeletes() async throws {305        let log = CallLog()306        let started = Signal()307        var handlers = registry(log)308        handlers.register(.photosScan) { _, _, _ in309            started.fire()310            try await Task.sleep(for: .seconds(60))311            return StepOutput(summary: "never")312        }313        let runner = WorkflowRunner()314        let definition = try cleanup()315        let clock = ContinuousClock()316        let begin = clock.now317        async let outcome = collect(runner, definition, handlers)318        await started.wait()319        await runner.cancel()320        let result = try await outcome321        #expect(clock.now - begin < .seconds(5))322        #expect(result.run.status == .cancelled)323        #expect(statuses(result.run) == [.cancelled, .notRun, .notRun, .notRun])324        #expect(result.events == ["started", "stepStarted(0)", "finished(cancelled)"])325        #expect(log.count("photos.delete") == 0 && log.count("photos.review") == 0)326    }327328    @Test func cancellationBeforeTheNextStepEvenWithAHandlerThatIgnoresIt() async throws {329        let log = CallLog()330        let started = Signal()331        let cancelled = Signal()332        var handlers = registry(log)333        // This handler does not check for cancellation: it finishes its work, then nothing else may start.334        handlers.register(.photosReview) { _, _, _ in335            started.fire()336            await cancelled.wait()337            return StepOutput(summary: "review finished anyway")338        }339        let runner = WorkflowRunner()340        let definition = try cleanup()341        async let outcome = collect(runner, definition, handlers)342        await started.wait()343        await runner.cancel()344        cancelled.fire()345        let result = try await outcome346        #expect(statuses(result.run) == [.succeeded, .succeeded, .notRun, .notRun])347        #expect(result.run.status == .cancelled)348        #expect(log.count("photos.delete") == 0)349        #expect(result.run.steps[2].startedAt == nil)350    }351352    @Test func cancellationWhileWaitingForConfirmationReturnsPromptly() async throws {353        let log = CallLog()354        let asked = Signal()355        let release = Signal()356        let runner = WorkflowRunner()357        let definition = try cleanup()358        let clock = ContinuousClock()359        let begin = clock.now360        // The dialog is left open: the closure does not return until the test releases it, long after the cancellation.361        async let outcome = collect(runner, definition, registry(log)) { _ in362            asked.fire()363            await release.wait()364            return .confirmed365        }366        await asked.wait()367        await runner.cancel()368        let result = try await outcome369        #expect(clock.now - begin < .seconds(5))370        #expect(statuses(result.run) == [.succeeded, .succeeded, .cancelled, .notRun])371        #expect(result.run.status == .cancelled)372        // The late "confirmed" answer changes nothing.373        release.fire()374        try await Task.sleep(for: .milliseconds(100))375        #expect(log.count("photos.delete") == 0)376        #expect(log.count("report.generate") == 0)377    }378379    @Test func cancellingTheConsumingTaskCancelsTheRun() async throws {380        let log = CallLog()381        let started = Signal()382        var handlers = registry(log)383        handlers.register(.photosScan) { _, _, _ in384            started.fire()385            try await Task.sleep(for: .seconds(60))386            return StepOutput(summary: "never")387        }388        let runner = WorkflowRunner()389        let definition = try cleanup()390        let consumer = Task {391            let stream = await runner.run(definition, handlers: handlers) { _ in .confirmed }392            for await _ in stream {}393        }394        await started.wait()395        consumer.cancel()396        await consumer.value397        // The record of the cancelled run arrives in the history.398        var history: [WorkflowRun] = []399        for _ in 0..<200 where history.isEmpty {400            try await Task.sleep(for: .milliseconds(10))401            history = await runner.history402        }403        let run = try #require(history.first)404        #expect(run.status == .cancelled)405        #expect(statuses(run) == [.cancelled, .notRun, .notRun, .notRun])406        #expect(log.count("photos.delete") == 0)407        let running = await runner.isRunning408        #expect(!running)409    }410411    @Test func cancellingOneRunLeavesTheOtherAlone() async throws {412        let log = CallLog()413        let started = Signal()414        var handlers = registry(log)415        handlers.register(.photosScan) { step, _, _ in416            if step.parameters["slow"] == "yes" {417                started.fire()418                try await Task.sleep(for: .seconds(60))419            }420            return StepOutput(summary: "scan")421        }422        let runner = WorkflowRunner()423        let slow = try WorkflowDefinition(name: "Slow", summary: "", steps: [step(.photosScan, parameters: ["slow": "yes"]), step(.reportGenerate)])424        let fast = try WorkflowDefinition(name: "Fast", summary: "", steps: [step(.photosScan), step(.reportGenerate)])425        async let slowOutcome = collect(runner, slow, handlers)426        await started.wait()427        let active = await runner.activeRunIDs428        let slowID = try #require(active.first)429        let fastResult = try await collect(runner, fast, handlers)430        #expect(fastResult.run.status == .succeeded)431        await runner.cancel(runID: slowID)432        let slowResult = try await slowOutcome433        #expect(slowResult.run.status == .cancelled && slowResult.run.id == slowID)434    }435436    @Test func failureStopsUnlessContinueOnFailure() async throws {437        let log = CallLog()438        var handlers = registry(log)439        handlers.register(.storageOverview) { _, _, _ in throw TestFailure() }440441        let stopping = try WorkflowDefinition(name: "S", summary: "", steps: [step(.securityAssessment), step(.storageOverview), step(.reportGenerate)])442        let stopped = try await collect(WorkflowRunner(), stopping, handlers)443        #expect(statuses(stopped.run) == [.succeeded, .failed, .notRun])444        #expect(stopped.run.status == .failed)445        #expect(stopped.run.steps[1].errorText == "The disk could not be measured.")446        #expect(stopped.events == ["started", "stepStarted(0)", "stepFinished(0)", "stepStarted(1)", "stepFailed(1)", "finished(failed)"])447        #expect(log.count("report.generate") == 0)448449        let continuing = try WorkflowDefinition(name: "C", summary: "", steps: [step(.securityAssessment), step(.storageOverview, continueOnFailure: true), step(.reportGenerate)])450        let continued = try await collect(WorkflowRunner(), continuing, handlers)451        #expect(statuses(continued.run) == [.succeeded, .failed, .succeeded])452        #expect(continued.run.status == .completedWithIssues)453        #expect(log.count("report.generate") == 1)454    }455456    @Test func unknownStepKindFailsWithAClearMessage() async throws {457        let log = CallLog()458        let definition = try WorkflowDefinition(name: "U", summary: "", steps: [step("battery.cycles"), step(.reportGenerate)])459        var events: [WorkflowEvent] = []460        let stream = await WorkflowRunner().run(definition, handlers: registry(log)) { _ in .confirmed }461        for await event in stream { events.append(event) }462        var message: String?463        var final: WorkflowRun?464        for event in events {465            if case .stepFailed(_, _, let error) = event { message = error }466            if case .finished(let run) = event { final = run }467        }468        let run = try #require(final)469        #expect(message == WorkflowError.unknownStepKind("battery.cycles").errorDescription)470        #expect(message?.contains("battery.cycles") == true)471        #expect(statuses(run) == [.failed, .notRun])472        #expect(run.status == .failed)473        #expect(log.all.isEmpty)474    }475476    @Test func stepTimeLimit() async throws {477        var handlers = WorkflowHandlerRegistry()478        handlers.register(.networkDiagnostics) { _, _, _ in479            try await Task.sleep(for: .seconds(60))480            return StepOutput(summary: "never")481        }482        handlers.register(.reportGenerate) { _, _, _ in StepOutput(summary: "report") }483        let definition = try WorkflowDefinition(name: "T", summary: "", steps: [step(.networkDiagnostics, continueOnFailure: true, timeout: 0.05), step(.reportGenerate, timeout: 30)])484        let clock = ContinuousClock()485        let begin = clock.now486        let result = try await collect(WorkflowRunner(), definition, handlers)487        #expect(clock.now - begin < .seconds(5))488        #expect(statuses(result.run) == [.failed, .succeeded])489        #expect(result.run.steps[0].errorText == WorkflowError.timedOut(seconds: 0.05).errorDescription)490    }491492    @Test func runnerRefusesADefinitionItsRegistryForbids() async throws {493        let log = CallLog()494        // Valid under the standard rules, but this registry declares `storage.overview` destructive.495        let definition = try WorkflowDefinition(name: "R", summary: "", steps: [step(.storageOverview), step(.reportGenerate)])496        let strict = WorkflowRules(destructiveKinds: [.storageOverview], requiredPredecessors: [.storageOverview: .checklistReview])497        let result = try await collect(WorkflowRunner(), definition, registry(log, rules: strict))498        #expect(result.events == ["started", "finished(failed)"])499        #expect(statuses(result.run) == [.notRun, .notRun])500        #expect(result.run.failureReason?.isEmpty == false)501        #expect(log.all.isEmpty)502    }503504    @Test func contextCarriesValuesBetweenSteps() async throws {505        var handlers = WorkflowHandlerRegistry()506        handlers.register(.photosScan) { step, context, _ in507            let seed = await context.value(for: "seed") ?? "none"508            await context.setValue("set-directly", for: "direct")509            return StepOutput(summary: "scanned", values: ["photos.count": "42", "echo": "\(seed)/\(step.parameters["album"] ?? "-")"])510        }511        handlers.register(.reportGenerate) { _, context, _ in512            let count = await context.value(for: "photos.count") ?? "?"513            let echo = await context.value(for: "echo") ?? "?"514            let direct = await context.value(for: "direct") ?? "?"515            let missing = await context.value(for: "missing")516            return StepOutput(summary: "report for \(count) photos, \(echo), \(direct), \(missing ?? "nil")")517        }518        let definition = try WorkflowDefinition(name: "Ctx", summary: "", steps: [step(.photosScan, parameters: ["album": "Recents"]), step(.reportGenerate)])519        let result = try await collect(WorkflowRunner(), definition, handlers, initialValues: ["seed": "S1"])520        #expect(result.run.steps[1].summary == "report for 42 photos, S1/Recents, set-directly, nil")521        #expect(result.run.values == ["seed": "S1", "photos.count": "42", "echo": "S1/Recents", "direct": "set-directly"])522    }523524    @Test func parallelRunsDoNotShareContext() async throws {525        var handlers = WorkflowHandlerRegistry()526        handlers.register(.photosScan) { step, context, _ in527            let previous = await context.value(for: "token")528            try await Task.sleep(for: .milliseconds(Int.random(in: 1...15)))529            return StepOutput(summary: previous ?? "clean", values: ["token": step.parameters["token"] ?? ""])530        }531        handlers.register(.reportGenerate) { _, context, _ in532            try await Task.sleep(for: .milliseconds(Int.random(in: 1...15)))533            let token = await context.value(for: "token") ?? "?"534            return StepOutput(summary: "\(context.runID.uuidString)=\(token)")535        }536        let frozen = handlers537        let runner = WorkflowRunner(historyLimit: 100)538        let results = try await withThrowingTaskGroup(of: (String, WorkflowRun).self) { group in539            for index in 0..<24 {540                group.addTask {541                    let token = "token-\(index)"542                    let definition = try WorkflowDefinition(name: "Run \(index)", summary: "", steps: [step(.photosScan, parameters: ["token": token]), step(.reportGenerate)])543                    // Half of the runs share one runner, the others have their own.544                    let result = try await collect(index % 2 == 0 ? runner : WorkflowRunner(), definition, frozen)545                    return (token, result.run)546                }547            }548            var all: [(String, WorkflowRun)] = []549            for try await result in group { all.append(result) }550            return all551        }552        #expect(results.count == 24)553        for (token, run) in results {554            #expect(run.status == .succeeded)555            #expect(run.steps[0].summary == "clean") // no value left by another run556            #expect(run.steps[1].summary == "\(run.id.uuidString)=\(token)")557            #expect(run.values == ["token": token])558        }559        #expect(Set(results.map(\.1.id)).count == 24)560        let history = await runner.history561        #expect(history.count == 12)562    }563564    @Test func historyIsBounded() async throws {565        let log = CallLog()566        let runner = WorkflowRunner(historyLimit: 3)567        var ids: [UUID] = []568        for _ in 0..<5 {569            let result = try await collect(runner, WorkflowLibrary.networkCheck, registry(log))570            ids.append(result.run.id)571        }572        var history = await runner.history573        #expect(history.map(\.id) == Array(ids.suffix(3)))574        await runner.clearHistory()575        history = await runner.history576        #expect(history.isEmpty)577    }578}579580// MARK: - Runs, reports, library581582@Suite("Workflow runs and library")583struct WorkflowRunTests {584    private func sampleFinding(_ id: String) -> Finding {585        Finding(id: id, category: .storage, title: "Finding \(id)", state: .attention, severity: .moderate, confidence: .measured, verification: .publicAPI, evidence: [Evidence("Free", "1 GB")], explanation: "Explanation \(id)", assessedAt: Date(timeIntervalSince1970: 1_795_176_000))586    }587588    private func mixedRun() async throws -> WorkflowRun {589        let inSection = sampleFinding("in-section")590        let loose = sampleFinding("loose")591        var handlers = WorkflowHandlerRegistry()592        handlers.register(.storageOverview) { _, _, _ in593            StepOutput(summary: "112 GB free", findings: [inSection, loose], section: .init(title: "Storage details", facts: [.init("Free", "112 GB")], findings: [inSection]))594        }595        handlers.register(.networkDiagnostics) { _, _, _ in throw TestFailure() }596        handlers.register(.reportGenerate) { _, _, _ in StepOutput(summary: "Report ready") }597        let definition = try WorkflowDefinition(name: "Mixed review", summary: "A mixed workflow.", steps: [598            step(.storageOverview), step(.networkDiagnostics, continueOnFailure: true), step("unknown.kind", continueOnFailure: true), step(.reportGenerate),599        ])600        let result = try await collect(WorkflowRunner(), definition, handlers)601        return result.run602    }603604    @Test func runReportContent() async throws {605        let run = try await mixedRun()606        #expect(run.status == .completedWithIssues)607        #expect(run.count(.succeeded) == 2 && run.count(.failed) == 2)608        #expect(run.findings.map(\.id) == ["in-section", "loose"])609        #expect(run.steps[0].findingCount == 2)610611        let report = run.report()612        #expect(report.kind == .workflow)613        #expect(report.title == "Mixed review")614        #expect(report.id == run.id)615        #expect(report.createdAt == run.endedAt)616        #expect(report.summary.contains("Mixed review") && report.summary.contains(run.status.title) && report.summary.contains("2 of 4"))617        #expect(report.sections.map(\.title) == ["Workflow", "Steps", "Storage details", "Findings"])618        let overview = report.sections[0]619        #expect(overview.paragraphs == ["A mixed workflow."])620        #expect(overview.facts.first == .init("Workflow", "Mixed review"))621        #expect(overview.facts.contains(.init("Result", run.status.title)))622        #expect(overview.facts.contains(.init("Steps completed", "2 of 4")))623        let steps = report.sections[1].facts624        #expect(steps.count == 4)625        #expect(steps[0].label == "1. Step storage.overview")626        #expect(steps[0].value.hasPrefix(WorkflowStepStatus.succeeded.title) && steps[0].value.hasSuffix("112 GB free"))627        #expect(steps[1].value.hasPrefix(WorkflowStepStatus.failed.title) && steps[1].value.contains("The disk could not be measured."))628        #expect(steps[2].value.contains("unknown.kind"))629        #expect(steps[3].value.hasSuffix("Report ready"))630        // A finding already shown in a step's section is not repeated.631        #expect(report.sections[2].findings.map(\.id) == ["in-section"])632        #expect(report.sections[3].findings.map(\.id) == ["loose"])633        #expect(report.allFindings.count == 2)634        // Steps that did not complete are stated as checks that could not be made.635        #expect(report.unavailable.count == 2)636        #expect(report.unavailable[0].contains("Step network.diagnostics") && report.unavailable[0].contains("The disk could not be measured."))637        #expect(run.report() == report)638    }639640    @Test func refusedAndCancelledRunsStillReport() async throws {641        let definition = try cleanup()642        let run = WorkflowRun(definition: definition, startedAt: Date(timeIntervalSince1970: 100), endedAt: Date(timeIntervalSince1970: 163.5), status: .cancelled, steps: definition.steps.map { WorkflowStepRecord(step: $0) })643        #expect(run.duration == 63.5)644        let report = run.report()645        #expect(report.unavailable.count == 4)646        #expect(report.sections[1].facts.allSatisfy { $0.value == WorkflowStepStatus.notRun.title })647        #expect(report.sections[0].facts.contains(.init("Result", WorkflowRun.Status.cancelled.title)))648        #expect(WorkflowStepStatus.allCases.allSatisfy { !$0.title.isEmpty })649        #expect(WorkflowRun.Status.allCases.allSatisfy { !$0.title.isEmpty })650    }651652    @Test func codableRoundTripOfDefinitionsAndRuns() async throws {653        let encoder = JSONEncoder()654        encoder.dateEncodingStrategy = .deferredToDate655        let decoder = JSONDecoder()656        for definition in WorkflowLibrary.builtIn {657            let data = try encoder.encode(definition)658            let decoded = try decoder.decode(WorkflowDefinition.self, from: data)659            #expect(decoded == definition)660        }661        let custom = try WorkflowDefinition(name: "Custom", summary: "S", systemImage: "star", steps: [step(.urlInspect, continueOnFailure: true, timeout: 12.5, parameters: ["url": "https://example.com/?a=1"])])662        let customData = try encoder.encode(custom)663        let customBack = try decoder.decode(WorkflowDefinition.self, from: customData)664        #expect(customBack == custom)665        let text = try #require(String(data: customData, encoding: .utf8))666        #expect(text.contains("\"kind\":\"url.inspect\"")) // kinds are plain strings in the file667668        let run = try await mixedRun()669        let runData = try encoder.encode(run)670        let runBack = try decoder.decode(WorkflowRun.self, from: runData)671        #expect(runBack == run)672        #expect(runBack.report() == run.report())673        let all = try encoder.encode([run, run])674        let allBack = try decoder.decode([WorkflowRun].self, from: all)675        #expect(allBack.count == 2)676    }677678    @Test func builtInLibraryIsValidAndUseful() throws {679        let library = WorkflowLibrary.builtIn680        #expect(library.count >= 5)681        #expect(Set(library.map(\.id)).count == library.count)682        #expect(Set(library.map(\.name)).count == library.count)683        #expect(library.map(\.id) == WorkflowLibrary.builtIn.map(\.id)) // stable identifiers684        #expect(library.flatMap(\.steps).map(\.id) == WorkflowLibrary.builtIn.flatMap(\.steps).map(\.id))685        #expect(Set(library.flatMap(\.steps).map(\.id)).count == library.flatMap(\.steps).count)686        let known: Set<WorkflowStepKind> = [.securityAssessment, .storageOverview, .photosScan, .photosReview, .photosDelete, .networkDiagnostics, .urlInspect, .checklistReview, .reportGenerate]687        for definition in library {688            #expect(throws: Never.self) { try definition.validate() }689            #expect(definition.isBuiltIn)690            #expect(!definition.name.isEmpty && !definition.summary.isEmpty && !definition.systemImage.isEmpty)691            #expect(definition.steps.allSatisfy { known.contains($0.kind) })692            #expect(definition.steps.last?.kind == .reportGenerate)693            for step in definition.steps where step.isDestructive { #expect(step.requiresConfirmation) }694        }695        let cleanup = WorkflowLibrary.storageCleanup696        #expect(cleanup.steps.map(\.kind) == [.storageOverview, .photosScan, .photosReview, .photosDelete, .reportGenerate])697        #expect(cleanup.steps[3].isDestructive && cleanup.steps[3].requiresConfirmation)698        #expect(WorkflowLibrary.securityCheckUp.steps.map(\.kind) == [.securityAssessment, .checklistReview, .networkDiagnostics, .reportGenerate])699        #expect(WorkflowLibrary.suspiciousLinkTriage.steps.map(\.kind) == [.urlInspect, .networkDiagnostics, .reportGenerate])700        // Only the cleanup workflow can delete anything.701        let destructive = library.filter { definition in definition.steps.contains { $0.isDestructive } }702        #expect(destructive.map(\.id) == [cleanup.id])703        #expect(WorkflowStepKind.photosDelete.rawValue == "photos.delete" && WorkflowStepKind("x.y") == "x.y")704    }705706    @Test func everyBuiltInWorkflowRunsToTheEnd() async throws {707        for definition in WorkflowLibrary.builtIn {708            let log = CallLog()709            let result = try await collect(WorkflowRunner(), definition, registry(log))710            #expect(result.run.status == .succeeded, "\(definition.name)")711            #expect(log.all == definition.steps.map(\.kind.rawValue))712        }713    }714}715
Files main
Menu

Theme