SPB Git

Repository Public License

SchedulerEngineTests.swift 41.6 KB 781 lines swift

1// =============================================================================2//                  SP BOUCHER | SOFTWARE & RESEARCH3//             ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project          : ORIX OS Automation9// File             : Packages/ORIXScheduler/Tests/ORIXSchedulerTests/SchedulerEngineTests.swift10// Description      : Engine tests with a manual clock and the in-memory store:11//                    dedup, missed-run policies, concurrency, retries,12//                    pause/resume/runNow/cancel, approval parking, lease13//                    takeover, run loop.14// Module           : ORIXSchedulerTests15// Repository       : https://git.spboucher.ai/orix-automation16// Created          : 2026-11-2117// Last Modified    : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author           : Simon-Pierre Boucher24// Position         : Professor of Finance25// Institution      : Université du Québec en Outaouais (UQO)26// Department       : Département des sciences administratives27// Organization     : SP Boucher28//29// Website          : https://spboucher.ai30// Git              : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General          : info@spboucher.ai37// Professional     : contact@spboucher.ai38// Direct           : simon@spboucher.ai39// Research         : research@spboucher.ai40// Development      : dev@spboucher.ai41// Support          : support@spboucher.ai42// Security/Admin   : admin@spboucher.ai43// Notifications    : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification   : Public70// Distribution     : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License          : Proprietary, all rights reserved (see license file)84// License File     : LICENSE85// Maintainer       : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Foundation93import Testing94@testable import ORIXScheduler9596/// Records launches and answers them from a script. Optionally holds every launch until `release()`.97final class LaunchRecorder: @unchecked Sendable {98    private let lock = NSLock()99    private var _launches: [QueueItem] = []100    private var waiting: [CheckedContinuation<Void, Never>] = []101    private var held: Bool102    private let answer: @Sendable (QueueItem) -> RunLaunchResult103104    init(held: Bool = false, answer: @escaping @Sendable (QueueItem) -> RunLaunchResult = { .succeeded(runID: "run-\($0.id.prefix(6))") }) {105        self.held = held106        self.answer = answer107    }108109    var launches: [QueueItem] { lock.withLock { _launches } }110    var count: Int { launches.count }111112    func launch(_ item: QueueItem, _ schedule: Schedule) async -> RunLaunchResult {113        await withCheckedContinuation { (c: CheckedContinuation<Void, Never>) in114            let go: Bool = lock.withLock {115                _launches.append(item)116                if held { waiting.append(c); return false }117                return true118            }119            if go { c.resume() }120        }121        return answer(item)122    }123124    /// Lets every held launch finish, and the following ones run straight through.125    func release() {126        let all: [CheckedContinuation<Void, Never>] = lock.withLock {127            held = false128            let w = waiting129            waiting = []130            return w131        }132        for c in all { c.resume() }133    }134}135136struct Harness {137    let clock: ManualSchedulerClock138    let store: InMemoryScheduleStore139    let recorder: LaunchRecorder140    let engine: SchedulerEngine141142    init(start: String, recorder: LaunchRecorder = LaunchRecorder(), owner: String = "engine-A",143         configuration: SchedulerConfiguration = SchedulerConfiguration(observesSystemWake: false)) {144        let clock = ManualSchedulerClock(now: utc(start))145        self.clock = clock146        store = InMemoryScheduleStore(now: { clock.now() })147        self.recorder = recorder148        engine = SchedulerEngine(store: store, clock: clock, launcher: { await recorder.launch($0, $1) }, owner: owner, configuration: configuration)149    }150151    func secondEngine(owner: String, recorder: LaunchRecorder) -> SchedulerEngine {152        SchedulerEngine(store: store, clock: clock, launcher: { await recorder.launch($0, $1) }, owner: owner,153                        configuration: SchedulerConfiguration(observesSystemWake: false))154    }155156    /// One tick, then wait for every launcher to finish.157    @discardableResult func step() async -> TickSummary {158        let summary = await engine.tick()159        await engine.settle()160        return summary161    }162163    func items(_ status: QueueItem.Status? = nil) async -> [QueueItem] {164        (try? await engine.queue(status: status)) ?? []165    }166}167168/// Daily at 09:00 Toronto. In June, 09:00 EDT is 13:00Z.169func dailyNine(_ policy: MissedRunPolicy = .skip, concurrency: ConcurrencyPolicy = .skipIfRunning, createdAt: String) -> Schedule {170    Schedule(id: "daily", workflowID: "wf-report", name: "Daily report", trigger: .cron(expression: "0 9 * * *", timezone: "America/Toronto"),171             missedRunPolicy: policy, concurrency: concurrency, createdAt: utc(createdAt))172}173174@Suite("Scheduler engine: firing and dedup (unit, manual clock, in-memory store)")175struct EngineFiringTests {176    @Test func firesWhenDueAndNotBefore() async throws {177        let h = Harness(start: "2026-06-01T12:00:00Z")178        let saved = try await h.engine.save(dailyNine(createdAt: "2026-06-01T12:00:00Z"))179        #expect(saved.nextFireAt == utc("2026-06-01T13:00:00Z"))180        var summary = await h.step()181        #expect(summary == TickSummary(active: true))182        h.clock.set(utc("2026-06-01T12:59:59Z"))183        #expect(await h.step().enqueued == 0)184        h.clock.set(utc("2026-06-01T13:00:00Z"))185        summary = await h.step()186        #expect(summary.enqueued == 1 && summary.launched == 1 && summary.skipped == 0)187        let items = await h.items()188        #expect(items.count == 1)189        #expect(items[0].status == .succeeded && items[0].attempt == 1 && items[0].runID != nil)190        #expect(items[0].dueAt == utc("2026-06-01T13:00:00Z"))191        #expect(items[0].dedupKey == "daily@\(Int64(utc("2026-06-01T13:00:00Z").timeIntervalSince1970 * 1000))")192        let schedule = try #require(try await h.engine.schedule(id: "daily"))193        #expect(schedule.lastFiredAt == utc("2026-06-01T13:00:00Z") && schedule.nextFireAt == utc("2026-06-02T13:00:00Z"))194        // Ticking again at the same instant changes nothing.195        #expect(await h.step().enqueued == 0)196        #expect(h.recorder.count == 1)197    }198199    @Test func aRestartNeverEnqueuesTheSameInstantTwice() async throws {200        let h = Harness(start: "2026-06-01T12:00:00Z")201        try await h.engine.save(dailyNine(.runAll(max: 5), createdAt: "2026-06-01T12:00:00Z"))202        h.clock.set(utc("2026-06-01T13:00:05Z"))203        await h.step()204        #expect(h.recorder.count == 1)205206        // Simulate a crash right after the enqueue: the schedule's cursor was never saved.207        var stale = try #require(await h.store.schedule(id: "daily"))208        stale.lastEvaluatedAt = utc("2026-06-01T12:00:00Z")209        stale.lastFiredAt = nil210        await h.store.saveSchedule(stale)211212        let afterRestart = LaunchRecorder()213        let restarted = h.secondEngine(owner: "engine-A", recorder: afterRestart)214        let summary = await restarted.tick()215        await restarted.settle()216        #expect(summary.active && summary.enqueued == 0 && summary.launched == 0)217        #expect(afterRestart.count == 0)218        #expect(await h.items().count == 1)219220        // Even long after: the instant is now "missed", the policy says run it, and the key still refuses a duplicate.221        h.clock.set(utc("2026-06-01T15:00:00Z"))222        stale.lastEvaluatedAt = utc("2026-06-01T12:00:00Z")223        await h.store.saveSchedule(stale)224        let later = await restarted.tick()225        await restarted.settle()226        #expect(later.enqueued == 0 && afterRestart.count == 0)227        #expect(await h.items().filter { $0.status != .skipped }.count == 1)228    }229230    @Test func theStoreRefusesASecondItemWithTheSameKey() async throws {231        let store = InMemoryScheduleStore()232        let now = utc("2026-06-01T13:00:00Z")233        let key = QueueItem.dedupKey(scheduleID: "s", instant: now)234        #expect(await store.enqueue(QueueItem(scheduleID: "s", workflowID: "w", dueAt: now, enqueuedAt: now, dedupKey: key)))235        #expect(await store.enqueue(QueueItem(scheduleID: "s", workflowID: "w", dueAt: now, enqueuedAt: now, dedupKey: key)) == false)236        #expect(await store.queueItem(dedupKey: key) != nil)237        #expect(await store.queueItems().count == 1)238        // Atomic claim: one claimer gets it, the next gets nothing.239        let claims = await withTaskGroup(of: QueueItem?.self) { group -> [QueueItem] in240            for _ in 0..<8 { group.addTask { try? await store.claimNext(now: now) } }241            var out: [QueueItem] = []242            for await c in group { if let c { out.append(c) } }243            return out244        }245        #expect(claims.count == 1 && claims[0].status == .running)246    }247248    @Test func onceAndIntervalTriggers() async throws {249        let h = Harness(start: "2026-06-01T12:00:00Z")250        try await h.engine.save(Schedule(id: "once", workflowID: "wf-a", name: "Once", trigger: .once(at: utc("2026-06-01T12:10:00Z")), createdAt: utc("2026-06-01T12:00:00Z")))251        try await h.engine.save(Schedule(id: "every", workflowID: "wf-b", name: "Every 4 min", trigger: .interval(seconds: 240), createdAt: utc("2026-06-01T12:00:00Z")))252        var launchedFor: [String: Int] = [:]253        for minute in 1...20 {254            h.clock.set(utc("2026-06-01T12:00:00Z").addingTimeInterval(Double(minute) * 60))255            await h.step()256        }257        for item in h.recorder.launches { launchedFor[item.scheduleID, default: 0] += 1 }258        #expect(launchedFor == ["once": 1, "every": 5])259        #expect(try await h.engine.schedule(id: "once")?.nextFireAt == nil)260        #expect(try await h.engine.schedule(id: "every")?.nextFireAt == utc("2026-06-01T12:24:00Z"))261        let everyDue = await h.items().filter { $0.scheduleID == "every" }.map(\.dueAt)262        #expect(everyDue == [4, 8, 12, 16, 20].map { utc("2026-06-01T12:00:00Z").addingTimeInterval($0 * 60) })263    }264265    @Test func invalidSchedulesAreRefused() async throws {266        let h = Harness(start: "2026-06-01T12:00:00Z")267        await #expect(throws: SchedulerError.self) {268            try await h.engine.save(Schedule(workflowID: "w", name: "bad", trigger: .cron(expression: "61 * * * *", timezone: nil)))269        }270        await #expect(throws: SchedulerError.self) {271            try await h.engine.save(Schedule(workflowID: "w", name: "bad", trigger: .cron(expression: "0 9 * * *", timezone: "Mars/Olympus")))272        }273        await #expect(throws: SchedulerError.self) {274            try await h.engine.save(Schedule(workflowID: "w", name: "bad", trigger: .interval(seconds: 0.1)))275        }276        await #expect(throws: SchedulerError.self) { try await h.engine.runNow(scheduleID: "nope") }277        #expect(try await h.engine.schedules().isEmpty)278    }279280    @Test func upcomingMergesSchedulesInOrder() async throws {281        let h = Harness(start: "2026-06-01T12:00:00Z")282        try await h.engine.save(dailyNine(createdAt: "2026-06-01T12:00:00Z"))283        try await h.engine.save(Schedule(id: "six-hourly", workflowID: "wf-b", name: "Six-hourly", trigger: .cron(expression: "0 */6 * * *", timezone: "UTC"), createdAt: utc("2026-06-01T12:00:00Z")))284        try await h.engine.save(Schedule(id: "paused", workflowID: "wf-c", name: "Paused", trigger: .interval(seconds: 60), paused: true, createdAt: utc("2026-06-01T12:00:00Z")))285        try await h.engine.save(Schedule(id: "manual", workflowID: "wf-d", name: "Manual", trigger: .manual, createdAt: utc("2026-06-01T12:00:00Z")))286        let upcoming = try await h.engine.upcoming(limit: 5)287        #expect(upcoming.map(\.0.id) == ["daily", "six-hourly", "six-hourly", "six-hourly", "six-hourly"])288        #expect(upcoming.map(\.1) == ["2026-06-01T13:00:00Z", "2026-06-01T18:00:00Z", "2026-06-02T00:00:00Z", "2026-06-02T06:00:00Z", "2026-06-02T12:00:00Z"].map(utc))289        #expect(try await h.engine.upcoming(limit: 0).isEmpty)290    }291}292293@Suite("Scheduler engine: missed runs after a three-day gap (unit, manual clock)")294struct EngineMissedRunTests {295    /// Runs on day 0 at 09:00, then the Mac is off until day 3 at 15:00 Toronto: 09:00 was missed on days 1, 2 and 3.296    private func afterGap(_ policy: MissedRunPolicy, wakeAt: String = "2026-06-04T19:00:00Z") async throws -> Harness {297        let h = Harness(start: "2026-06-01T12:00:00Z")298        try await h.engine.save(dailyNine(policy, concurrency: .queue, createdAt: "2026-06-01T12:00:00Z"))299        h.clock.set(utc("2026-06-01T13:00:00Z"))300        await h.step()301        #expect(h.recorder.count == 1)302        h.clock.set(utc(wakeAt))303        return h304    }305306    @Test func skipRunsNothingAndSaysWhatWasMissed() async throws {307        let h = try await afterGap(.skip)308        let summary = await h.step()309        #expect(summary.enqueued == 0 && summary.launched == 0 && summary.skipped == 1)310        #expect(h.recorder.count == 1)311        let skipped = await h.items(.skipped)312        #expect(skipped.count == 1)313        #expect(skipped[0].reason.contains("3 scheduled firing(s)") && skipped[0].reason.contains("3 not run (policy skip)"))314        #expect(skipped[0].reason.contains("2026-06-02T13:00:00Z") && skipped[0].reason.contains("2026-06-04T13:00:00Z"))315        #expect(skipped[0].runID == nil)316        // It never claims a run for the missed days, and does not repeat the record.317        #expect(await h.items(.succeeded).count == 1)318        #expect(await h.step().skipped == 0)319        #expect(try await h.engine.schedule(id: "daily")?.nextFireAt == utc("2026-06-05T13:00:00Z"))320    }321322    @Test func runOnceRunsTheLatestMissedInstantOnly() async throws {323        let h = try await afterGap(.runOnce)324        let summary = await h.step()325        #expect(summary.enqueued == 1 && summary.launched == 1 && summary.skipped == 1)326        #expect(h.recorder.count == 2)327        let caughtUp = h.recorder.launches[1]328        #expect(caughtUp.dueAt == utc("2026-06-04T13:00:00Z") && caughtUp.reason.hasPrefix("catch-up"))329        #expect(await h.items(.skipped).first?.reason.contains("2 not run (policy runOnce)") == true)330        #expect(await h.step() == TickSummary(active: true))331    }332333    @Test func runOnceDoesNotDoubleAnOnTimeRun() async throws {334        // Wakes 30 s after day 3's 09:00: that one is on time; days 1 and 2 were missed.335        let h = try await afterGap(.runOnce, wakeAt: "2026-06-04T13:00:30Z")336        let summary = await h.step()337        #expect(summary.enqueued == 1 && summary.launched == 1 && summary.skipped == 1)338        #expect(h.recorder.launches.last?.reason.hasPrefix("scheduled for") == true)339        #expect(await h.items(.skipped).first?.reason.contains("2 not run (policy runOnce (covered by the on-time run))") == true)340    }341342    @Test func runAllIsBoundedByMax() async throws {343        let two = try await afterGap(.runAll(max: 2))344        var summary = await two.step()345        #expect(summary.enqueued == 2 && summary.skipped == 1)346        #expect(two.recorder.launches.dropFirst().map(\.dueAt) == ["2026-06-03T13:00:00Z", "2026-06-04T13:00:00Z"].map(utc))347        #expect(await two.items(.skipped).first?.reason.contains("1 not run (policy runAll(max: 2))") == true)348349        let ten = try await afterGap(.runAll(max: 10))350        summary = await ten.step()351        #expect(summary.enqueued == 3 && summary.skipped == 0)352        #expect(ten.recorder.launches.dropFirst().map(\.dueAt) == ["2026-06-02T13:00:00Z", "2026-06-03T13:00:00Z", "2026-06-04T13:00:00Z"].map(utc))353        #expect(await ten.items(.skipped).isEmpty)354        #expect(await ten.items(.succeeded).count == 4)355    }356357    @Test func aBusyEveryMinuteScheduleIsSummarisedNotReplayed() async throws {358        let h = Harness(start: "2026-06-01T12:00:00Z")359        try await h.engine.save(Schedule(id: "minutely", workflowID: "wf", name: "Every minute", trigger: .cron(expression: "* * * * *", timezone: "UTC"),360                                         missedRunPolicy: .runOnce, concurrency: .queue, createdAt: utc("2026-06-01T12:00:00Z")))361        h.clock.set(utc("2026-06-04T12:00:30Z")) // three days later362        let summary = await h.step()363        // 4320 instants passed; the last two (11:59, 12:00) are within the tolerance and run on time.364        #expect(summary.enqueued == 2 && summary.skipped == 1)365        let reason = try #require(await h.items(.skipped).first?.reason)366        #expect(reason.contains("4318 scheduled firing(s)"), "\(reason)")367    }368}369370@Suite("Scheduler engine: concurrency, retries and control (unit, manual clock)")371struct EngineControlTests {372    private func everyMinute(_ concurrency: ConcurrencyPolicy, maxRetries: Int = 0, backoff: Double = 10) -> Schedule {373        Schedule(id: "s", workflowID: "wf", name: "Every minute", trigger: .interval(seconds: 60), concurrency: concurrency,374                 maxRetries: maxRetries, retryBackoffSeconds: backoff, createdAt: utc("2026-06-01T12:00:00Z"))375    }376377    private func advance(_ h: Harness, toMinute m: Double) { h.clock.set(utc("2026-06-01T12:00:00Z").addingTimeInterval(m * 60)) }378379    @Test func skipIfRunningDropsOverlappingFirings() async throws {380        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))381        try await h.engine.save(everyMinute(.skipIfRunning))382        advance(h, toMinute: 1)383        #expect(await h.engine.tick().launched == 1)384        advance(h, toMinute: 2)385        let second = await h.engine.tick()386        #expect(second.enqueued == 1 && second.launched == 0 && second.skipped == 1)387        #expect(await h.items(.running).count == 1)388        #expect(await h.items(.skipped).first?.reason.contains("skipIfRunning") == true)389        h.recorder.release()390        await h.engine.settle()391        advance(h, toMinute: 3)392        await h.step()393        #expect(h.recorder.count == 2)394        #expect(await h.items(.succeeded).count == 2)395    }396397    @Test func queueHoldsOverlappingFiringsUntilTheWorkflowIsFree() async throws {398        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))399        try await h.engine.save(everyMinute(.queue))400        for m in 1...3 {401            advance(h, toMinute: Double(m))402            await h.engine.tick()403        }404        #expect(await eventually { h.recorder.count == 1 })405        #expect(await h.items(.running).count == 1)406        #expect(await h.items(.pending).count == 2)407        // The per-workflow lock also covers another schedule of the same workflow.408        try await h.engine.save(Schedule(id: "other", workflowID: "wf", name: "Other", trigger: .manual, concurrency: .queue, createdAt: utc("2026-06-01T12:00:00Z")))409        h.clock.advance(by: 1)410        try await h.engine.runNow(scheduleID: "other")411        #expect(await h.engine.tick().launched == 0)412        // Completion starts the next one without waiting for a tick, one at a time, oldest first.413        h.recorder.release()414        await h.engine.settle()415        #expect(h.recorder.count == 4)416        #expect(h.recorder.launches.map(\.scheduleID) == ["s", "s", "s", "other"])417        #expect(await h.items(.succeeded).count == 4)418    }419420    @Test func parallelAllowsUpToMaxRuns() async throws {421        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))422        try await h.engine.save(everyMinute(.parallel(max: 2)))423        for m in 1...3 {424            advance(h, toMinute: Double(m))425            await h.engine.tick()426        }427        #expect(await eventually { h.recorder.count == 2 })428        #expect(await h.items(.running).count == 2)429        #expect(await h.items(.pending).count == 1)430        h.recorder.release()431        await h.engine.settle()432        #expect(h.recorder.count == 3)433    }434435    @Test func theGlobalLimitCapsRunsAcrossWorkflows() async throws {436        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true),437                        configuration: SchedulerConfiguration(maxConcurrentRuns: 2, observesSystemWake: false))438        for i in 0..<4 {439            try await h.engine.save(Schedule(id: "s\(i)", workflowID: "wf\(i)", name: "S\(i)", trigger: .manual, createdAt: utc("2026-06-01T12:00:00Z")))440            try await h.engine.runNow(scheduleID: "s\(i)")441        }442        #expect(await h.engine.tick().launched == 2)443        #expect(await h.items(.pending).count == 2)444        h.recorder.release()445        await h.engine.settle()446        #expect(h.recorder.count == 4)447    }448449    @Test func retriesOnlyRetryableFailuresWithDoublingBackoff() async throws {450        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .failed(runID: "r", error: "network down", retryable: true) }))451        try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Once", trigger: .once(at: utc("2026-06-01T12:01:00Z")),452                                         maxRetries: 2, retryBackoffSeconds: 10, createdAt: utc("2026-06-01T12:00:00Z")))453        advance(h, toMinute: 1)454        await h.step()455        var item = try #require(await h.items().first)456        #expect(item.status == .pending && item.attempt == 2 && item.lastError == "network down")457        #expect(item.dueAt == utc("2026-06-01T12:01:10Z"))458        // Not before the backoff has elapsed.459        h.clock.advance(by: 9)460        #expect(await h.step().launched == 0)461        h.clock.advance(by: 1)462        #expect(await h.step().launched == 1)463        item = try #require(await h.items().first)464        #expect(item.status == .pending && item.attempt == 3 && item.dueAt == utc("2026-06-01T12:01:30Z"))465        h.clock.advance(by: 20)466        await h.step()467        item = try #require(await h.items().first)468        #expect(item.status == .failed && item.attempt == 3)469        #expect(h.recorder.count == 3)470        h.clock.advance(by: 3600)471        await h.step()472        #expect(h.recorder.count == 3)473    }474475    @Test func nonRetryableFailuresAndZeroRetriesAreFinal() async throws {476        let notRetryable = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .failed(runID: nil, error: "mail already sent", retryable: false) }))477        try await notRetryable.engine.save(everyMinute(.queue, maxRetries: 5))478        advance(notRetryable, toMinute: 1)479        await notRetryable.step()480        #expect(await notRetryable.items(.failed).count == 1)481        #expect(notRetryable.recorder.count == 1)482483        let noRetries = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .failed(runID: nil, error: "boom", retryable: true) }))484        try await noRetries.engine.save(everyMinute(.queue, maxRetries: 0))485        advance(noRetries, toMinute: 1)486        await noRetries.step()487        #expect(await noRetries.items(.failed).first?.lastError == "boom")488        #expect(noRetries.recorder.count == 1)489    }490491    @Test func backoffIsCapped() async throws {492        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .failed(runID: nil, error: "x", retryable: true) }),493                        configuration: SchedulerConfiguration(maxBackoffSeconds: 25, observesSystemWake: false))494        try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Manual", trigger: .manual, maxRetries: 4, retryBackoffSeconds: 10, createdAt: utc("2026-06-01T12:00:00Z")))495        try await h.engine.runNow(scheduleID: "s")496        var delays: [Double] = []497        for _ in 0..<4 {498            await h.step()499            let item = try #require(await h.items().first)500            delays.append(item.dueAt.timeIntervalSince(h.clock.now()))501            h.clock.set(item.dueAt)502        }503        #expect(delays == [10, 20, 25, 25])504    }505506    @Test func pauseHoldsAndResumeDoesNotCatchUp() async throws {507        let h = Harness(start: "2026-06-01T12:00:00Z")508        try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Every minute", trigger: .interval(seconds: 60),509                                         missedRunPolicy: .runAll(max: 100), createdAt: utc("2026-06-01T12:00:00Z")))510        try await h.engine.pause(scheduleID: "s")511        #expect(try await h.engine.schedule(id: "s")?.nextFireAt == nil)512        advance(h, toMinute: 30)513        #expect(await h.step() == TickSummary(active: true))514        try await h.engine.resume(scheduleID: "s")515        #expect(try await h.engine.schedule(id: "s")?.nextFireAt == utc("2026-06-01T12:31:00Z"))516        #expect(await h.step().enqueued == 0)517        advance(h, toMinute: 31)518        #expect(await h.step().enqueued == 1)519        #expect(h.recorder.count == 1)520    }521522    @Test func runNowWorksWhilePausedAndIsNotDeduplicated() async throws {523        let h = Harness(start: "2026-06-01T12:00:00Z")524        try await h.engine.save(everyMinute(.queue))525        try await h.engine.pause(scheduleID: "s")526        let first = try await h.engine.runNow(scheduleID: "s")527        let second = try await h.engine.runNow(scheduleID: "s")528        #expect(first.dedupKey != second.dedupKey && first.reason == "manual: run now")529        await h.step()530        #expect(h.recorder.count == 2)531        #expect(await h.items(.succeeded).count == 2)532        #expect(try await h.engine.schedule(id: "s")?.lastFiredAt == utc("2026-06-01T12:00:00Z"))533    }534535    @Test func cancelPendingRunningAndFinishedItems() async throws {536        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))537        try await h.engine.save(everyMinute(.queue))538        advance(h, toMinute: 1)539        await h.engine.tick()540        advance(h, toMinute: 2)541        await h.engine.tick()542        let running = try #require(await h.items(.running).first)543        let pending = try #require(await h.items(.pending).first)544        try await h.engine.cancel(queueItemID: pending.id)545        try await h.engine.cancel(queueItemID: running.id)546        #expect(await h.items(.canceled).count == 2)547        // The launcher's late "succeeded" does not resurrect the canceled run.548        h.recorder.release()549        await h.engine.settle()550        #expect(await h.items(.canceled).count == 2)551        #expect(await h.items(.succeeded).isEmpty)552        #expect(h.recorder.count == 1)553        // Canceling a finished item is a no-op; an unknown one is an error.554        try await h.engine.cancel(queueItemID: running.id)555        await #expect(throws: SchedulerError.self) { try await h.engine.cancel(queueItemID: "nope") }556    }557558    @Test func disableCancelsPendingAndStopsFiring() async throws {559        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))560        try await h.engine.save(everyMinute(.queue))561        for m in 1...3 { advance(h, toMinute: Double(m)); await h.engine.tick() }562        try await h.engine.disable(scheduleID: "s")563        #expect(await h.items(.canceled).map(\.reason) == ["schedule disabled", "schedule disabled"])564        h.recorder.release()565        await h.engine.settle()566        advance(h, toMinute: 10)567        #expect(await h.step() == TickSummary(active: true))568        #expect(h.recorder.count == 1)569        try await h.engine.enable(scheduleID: "s")570        advance(h, toMinute: 11)571        #expect(await h.step().enqueued == 1)572        // Removing the schedule cancels what is still pending.573        try await h.engine.pause(scheduleID: "s")574        _ = try await h.engine.fire(scheduleID: "s", reason: "ignored while paused")575        try await h.engine.resume(scheduleID: "s")576        advance(h, toMinute: 12)577        try await h.engine.remove(scheduleID: "s")578        #expect(await h.step().launched == 0)579    }580581    @Test func eventFiringRespectsEnabledPausedAndDedupKeys() async throws {582        let h = Harness(start: "2026-06-01T12:00:00Z")583        try await h.engine.save(Schedule(id: "login", workflowID: "wf", name: "At login", trigger: .login, concurrency: .queue, createdAt: utc("2026-06-01T12:00:00Z")))584        #expect(try await h.engine.fire(scheduleID: "login", reason: "login", dedupKey: "login@login-1") != nil)585        #expect(try await h.engine.fire(scheduleID: "login", reason: "login", dedupKey: "login@login-1") == nil)586        #expect(try await h.engine.fire(scheduleID: "missing", reason: "x") == nil)587        try await h.engine.pause(scheduleID: "login")588        #expect(try await h.engine.fire(scheduleID: "login", reason: "x") == nil)589        try await h.engine.resume(scheduleID: "login")590        try await h.engine.disable(scheduleID: "login")591        #expect(try await h.engine.fire(scheduleID: "login", reason: "x") == nil)592        #expect(await h.items().count == 1)593    }594}595596@Suite("Scheduler engine: approvals, lease and run loop (unit, manual clock)")597struct EngineApprovalLeaseTests {598    @Test func waitingApprovalItemsStayParkedAndVisible() async throws {599        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .waitingApproval(runID: "run-42") }))600        try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Every minute", trigger: .interval(seconds: 60), createdAt: utc("2026-06-01T12:00:00Z")))601        h.clock.advance(by: 60)602        await h.step()603        let parked = try #require(await h.items(.waitingApproval).first)604        #expect(parked.runID == "run-42" && parked.reason.contains("waiting for approval"))605        // Hours of ticking never approve, relaunch or drop it; the workflow stays locked.606        for _ in 0..<5 {607            h.clock.advance(by: 3600)608            await h.step()609        }610        #expect(h.recorder.count == 1)611        #expect(await h.items(.waitingApproval).map(\.id) == [parked.id])612        #expect(await h.items(.succeeded).isEmpty)613        #expect(await h.items(.skipped).contains { $0.reason.contains("skipIfRunning") })614        // Only an explicit resolution ends it.615        await #expect(throws: SchedulerError.self) { try await h.engine.resolveApproval(queueItemID: "nope", result: .succeeded(runID: nil)) }616        try await h.engine.resolveApproval(queueItemID: parked.id, result: .succeeded(runID: "run-42"))617        #expect(await h.store.queueItem(id: parked.id)?.status == .succeeded)618        await #expect(throws: SchedulerError.self) { try await h.engine.resolveApproval(queueItemID: parked.id, result: .succeeded(runID: nil)) }619    }620621    @Test func aParkedItemCanBeCanceled() async throws {622        let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .waitingApproval(runID: nil) }))623        try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Manual", trigger: .manual, createdAt: utc("2026-06-01T12:00:00Z")))624        let item = try await h.engine.runNow(scheduleID: "s")625        await h.step()626        try await h.engine.cancel(queueItemID: item.id)627        #expect(await h.store.queueItem(id: item.id)?.status == .canceled)628    }629630    @Test func aSecondEngineStaysPassiveThenTakesOverWhenTheLeaseExpires() async throws {631        let h = Harness(start: "2026-06-01T12:00:00Z")632        let recorderB = LaunchRecorder()633        let b = h.secondEngine(owner: "engine-B", recorder: recorderB)634        try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Every minute", trigger: .interval(seconds: 60),635                                         missedRunPolicy: .runAll(max: 10), createdAt: utc("2026-06-01T12:00:00Z")))636        #expect(await h.engine.tick().active)637        #expect(await b.tick().active == false)638        #expect(await b.isActive == false)639        #expect(await h.store.leaseOwner(name: "orix.scheduler") == "engine-A")640641        // A keeps renewing: B stays passive even when something is due.642        h.clock.advance(by: 20)643        await h.engine.tick()644        h.clock.advance(by: 20)645        await h.engine.tick()646        h.clock.advance(by: 20) // 12:01:00, due647        let passive = await b.tick()648        #expect(passive == TickSummary(active: false))649        #expect(recorderB.count == 0)650        await h.step()651        #expect(h.recorder.count == 1)652653        // A dies. Its lease (30 s) runs out; B takes over and handles what A never did.654        h.clock.advance(by: 29)655        #expect(await b.tick().active == false)656        h.clock.advance(by: 32) // 12:02:01657        let takeover = await b.tick()658        await b.settle()659        #expect(takeover.active && takeover.enqueued == 1)660        #expect(recorderB.count == 1 && h.recorder.count == 1)661        #expect(await h.store.leaseOwner(name: "orix.scheduler") == "engine-B")662        // A comes back: it is the passive one now, and no instant ran twice.663        #expect(await h.engine.tick().active == false)664        #expect(await h.engine.isActive == false)665        let due = await h.items(.succeeded).map(\.dueAt)666        #expect(due == [utc("2026-06-01T12:01:00Z"), utc("2026-06-01T12:02:00Z")])667    }668669    @Test func stopReleasesTheLeaseImmediately() async throws {670        let h = Harness(start: "2026-06-01T12:00:00Z")671        let b = h.secondEngine(owner: "engine-B", recorder: LaunchRecorder())672        await h.engine.tick()673        #expect(await b.tick().active == false)674        await h.engine.stop()675        #expect(await b.tick().active)676    }677678    @Test func runsInterruptedByARestartAreMarkedFailedAndNotRelaunched() async throws {679        let h = Harness(start: "2026-06-01T12:00:00Z")680        try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Manual", trigger: .manual, maxRetries: 3, createdAt: utc("2026-06-01T12:00:00Z")))681        let now = h.clock.now()682        _ = await h.store.enqueue(QueueItem(id: "orphan", scheduleID: "s", workflowID: "wf", dueAt: now, enqueuedAt: now, status: .running, dedupKey: "s@orphan"))683        let summary = await h.step()684        #expect(summary.launched == 0 && h.recorder.count == 0)685        let orphan = try #require(await h.store.queueItem(id: "orphan"))686        #expect(orphan.status == .failed && orphan.lastError?.contains("interrupted") == true)687    }688689    @Test func theRunLoopSleepsUntilTheNextInstantAndWakesOnChanges() async throws {690        let h = Harness(start: "2026-06-01T12:00:00Z", configuration: SchedulerConfiguration(heartbeatSeconds: 3600, observesSystemWake: false))691        try await h.engine.save(dailyNine(createdAt: "2026-06-01T12:00:00Z"))692        let loop = Task { await h.engine.run() }693        defer { loop.cancel() }694695        func eventually(_ what: String, _ condition: @Sendable () async -> Bool) async {696            for _ in 0..<400 {697                if await condition() { return }698                try? await Task.sleep(nanoseconds: 10_000_000)699            }700            Issue.record("timed out waiting for: \(what)")701        }702        // The loop parks on the clock; nothing fires early.703        await eventually("loop asleep") { h.clock.sleeperCount == 1 }704        #expect(h.recorder.count == 0)705        // Reaching the instant wakes it without any poke.706        h.clock.set(utc("2026-06-01T13:00:00Z"))707        await eventually("scheduled run") { await h.items(.succeeded).count == 1 }708        // A change made through the engine wakes it at once, with the clock standing still.709        await eventually("loop asleep again") { h.clock.sleeperCount == 1 }710        try await h.engine.runNow(scheduleID: "daily")711        await eventually("manual run") { await h.items(.succeeded).count == 2 }712        // So does an outside store change announced with storeDidChange().713        await eventually("loop asleep again") { h.clock.sleeperCount == 1 }714        let now = h.clock.now()715        _ = await h.store.enqueue(QueueItem(scheduleID: "daily", workflowID: "wf-report", dueAt: now, enqueuedAt: now, dedupKey: "daily@external"))716        h.engine.storeDidChange()717        await eventually("external item") { await h.items(.succeeded).count == 3 }718        await h.engine.stop()719        await eventually("loop ended") { h.clock.sleeperCount == 0 }720        _ = await loop.value721    }722}723724@Suite("Scheduler models (unit)")725struct SchedulerModelTests {726    @Test func modelsRoundTripThroughJSON() throws {727        let triggers: [TriggerSpec] = [728            .cron(expression: "0 9 * * MON", timezone: "America/Toronto"), .cron(expression: "@daily", timezone: nil),729            .once(at: utc("2026-06-01T12:00:00Z")), .interval(seconds: 90),730            .fileChange(path: "/tmp/in", recursive: true, glob: "*.pdf", debounceSeconds: 2), .login,731            .appLaunch(bundleID: "com.apple.Safari"), .powerConnected, .networkAvailable, .manual,732        ]733        let encoder = JSONEncoder(), decoder = JSONDecoder()734        for (i, trigger) in triggers.enumerated() {735            let schedule = Schedule(workflowID: "wf", name: "S\(i)", trigger: trigger, missedRunPolicy: i % 2 == 0 ? .runAll(max: 3) : .runOnce,736                                    concurrency: i % 2 == 0 ? .parallel(max: 2) : .queue, maxRetries: 2, lastFiredAt: utc("2026-06-01T12:00:00Z"))737            #expect(try decoder.decode(Schedule.self, from: encoder.encode(schedule)) == schedule)738            #expect(schedule.problems().isEmpty)739        }740        let item = QueueItem(scheduleID: "s", workflowID: "wf", dueAt: utc("2026-06-01T12:00:00Z"), enqueuedAt: utc("2026-06-01T12:00:01Z"),741                             attempt: 2, status: .waitingApproval, dedupKey: "s@1", reason: "r", runID: "run", lastError: "e")742        #expect(try decoder.decode(QueueItem.self, from: encoder.encode(item)) == item)743        #expect(Set(QueueItem.Status.allCases.map(\.rawValue)) == ["pending", "running", "succeeded", "failed", "skipped", "canceled", "waitingApproval"])744        #expect(QueueItem.Status.allCases.filter { !$0.isTerminal } == [.pending, .running, .waitingApproval])745    }746747    @Test func triggerMathForIntervalAndOnce() {748        let anchor = utc("2026-06-01T12:00:00Z")749        let every = TriggerSpec.interval(seconds: 90)750        #expect(every.nextFireDate(after: anchor, anchor: anchor) == anchor.addingTimeInterval(90))751        #expect(every.nextFireDate(after: anchor.addingTimeInterval(90), anchor: anchor) == anchor.addingTimeInterval(180))752        #expect(every.nextFireDate(after: anchor.addingTimeInterval(-500), anchor: anchor) == anchor.addingTimeInterval(90))753        #expect(every.latestFireDate(after: anchor, atOrBefore: anchor.addingTimeInterval(400), anchor: anchor) == anchor.addingTimeInterval(360))754        #expect(every.latestFireDate(after: anchor, atOrBefore: anchor.addingTimeInterval(89), anchor: anchor) == nil)755        let once = TriggerSpec.once(at: anchor.addingTimeInterval(10))756        #expect(once.nextFireDate(after: anchor, anchor: anchor) == anchor.addingTimeInterval(10))757        #expect(once.nextFireDate(after: anchor.addingTimeInterval(10), anchor: anchor) == nil)758        #expect(once.latestFireDate(after: anchor, atOrBefore: anchor.addingTimeInterval(60), anchor: anchor) == anchor.addingTimeInterval(10))759        #expect(TriggerSpec.manual.nextFireDate(after: anchor, anchor: anchor) == nil && !TriggerSpec.login.isTimeBased)760        #expect(TriggerSpec.cron(expression: "bad", timezone: nil).problems().count == 1)761        #expect(Schedule(workflowID: "", name: "x", trigger: .manual, missedRunPolicy: .runAll(max: 0), concurrency: .parallel(max: 0), maxRetries: -1).problems().count == 4)762        #expect(ConcurrencyPolicy.parallel(max: 3).capacity == 3 && ConcurrencyPolicy.queue.capacity == 1)763    }764765    @Test func manualClockSleepersWakeInOrderAndHonourCancellation() async throws {766        let clock = ManualSchedulerClock(now: utc("2026-06-01T12:00:00Z"))767        try await clock.sleep(until: utc("2026-06-01T11:00:00Z")) // already past768        let sleeper = Task { try await clock.sleep(until: utc("2026-06-01T12:00:10Z")); return clock.now() }769        while clock.sleeperCount == 0 { await Task.yield() }770        clock.advance(by: 5)771        #expect(clock.sleeperCount == 1)772        clock.advance(by: 5)773        #expect(try await sleeper.value == utc("2026-06-01T12:00:10Z"))774        let canceled = Task { try await clock.sleep(until: utc("2027-01-01T00:00:00Z")) }775        while clock.sleeperCount == 0 { await Task.yield() }776        canceled.cancel()777        await #expect(throws: CancellationError.self) { try await canceled.value }778        #expect(clock.sleeperCount == 0)779    }780}781
Files main
Menu

Theme