Repository Public License
1// =============================================================================2// SP BOUCHER | SOFTWARE & RESEARCH3// ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING4// =============================================================================5//6// PROJECT METADATA7// -----------------------------------------------------------------------------8// Project : ORIX OS Automation9// File : Packages/ORIXScheduler/Tests/ORIXSchedulerTests/SchedulerEngineTests.swift10// Description : Engine tests with a manual clock and the in-memory store:11// dedup, missed-run policies, concurrency, retries,12// pause/resume/runNow/cancel, approval parking, lease13// takeover, run loop.14// Module : ORIXSchedulerTests15// Repository : https://git.spboucher.ai/orix-automation16// Created : 2026-11-2117// Last Modified : 2026-11-2118//19// =============================================================================20// AUTHOR & PROFESSIONAL AFFILIATION21// =============================================================================22//23// Author : Simon-Pierre Boucher24// Position : Professor of Finance25// Institution : Université du Québec en Outaouais (UQO)26// Department : Département des sciences administratives27// Organization : SP Boucher28//29// Website : https://spboucher.ai30// Git : https://git.spboucher.ai31//32// =============================================================================33// OFFICIAL CONTACT DIRECTORY34// =============================================================================35//36// General : info@spboucher.ai37// Professional : contact@spboucher.ai38// Direct : simon@spboucher.ai39// Research : research@spboucher.ai40// Development : dev@spboucher.ai41// Support : support@spboucher.ai42// Security/Admin : admin@spboucher.ai43// Notifications : notifications@spboucher.ai44//45// Preferred Contact: support@spboucher.ai46//47// =============================================================================48// COPYRIGHT & INTELLECTUAL PROPERTY49// =============================================================================50//51// Copyright (c) 202652// Simon-Pierre Boucher53//54// All rights reserved where applicable.55//56// This file is governed by the applicable project license,57// copyright notices, institutional agreements and relevant law.58//59// Unless expressly authorized by the applicable license or60// rights holder, copying, redistribution, sublicensing,61// publication and commercial exploitation are not permitted.62//63// Existing third-party rights and notices remain unaffected.64//65// =============================================================================66// CONFIDENTIALITY & SECURITY67// =============================================================================68//69// Classification : Public70// Distribution : Public repository; use is governed by the project license71// Security Contact : admin@spboucher.ai72//73// Access to this source does not independently grant permission74// to redistribute or disclose confidential materials.75//76// Do not embed credentials, private keys, access tokens,77// certificates or confidential personal information.78//79// =============================================================================80// LICENSE & MAINTENANCE81// =============================================================================82//83// License : Proprietary, all rights reserved (see license file)84// License File : LICENSE85// Maintainer : Simon-Pierre Boucher86// Technical Contact: dev@spboucher.ai87//88// =============================================================================89// END OF SP BOUCHER SOURCE HEADER90// =============================================================================9192import Foundation93import Testing94@testable import ORIXScheduler9596/// Records launches and answers them from a script. Optionally holds every launch until `release()`.97final class LaunchRecorder: @unchecked Sendable {98 private let lock = NSLock()99 private var _launches: [QueueItem] = []100 private var waiting: [CheckedContinuation<Void, Never>] = []101 private var held: Bool102 private let answer: @Sendable (QueueItem) -> RunLaunchResult103104 init(held: Bool = false, answer: @escaping @Sendable (QueueItem) -> RunLaunchResult = { .succeeded(runID: "run-\($0.id.prefix(6))") }) {105 self.held = held106 self.answer = answer107 }108109 var launches: [QueueItem] { lock.withLock { _launches } }110 var count: Int { launches.count }111112 func launch(_ item: QueueItem, _ schedule: Schedule) async -> RunLaunchResult {113 await withCheckedContinuation { (c: CheckedContinuation<Void, Never>) in114 let go: Bool = lock.withLock {115 _launches.append(item)116 if held { waiting.append(c); return false }117 return true118 }119 if go { c.resume() }120 }121 return answer(item)122 }123124 /// Lets every held launch finish, and the following ones run straight through.125 func release() {126 let all: [CheckedContinuation<Void, Never>] = lock.withLock {127 held = false128 let w = waiting129 waiting = []130 return w131 }132 for c in all { c.resume() }133 }134}135136struct Harness {137 let clock: ManualSchedulerClock138 let store: InMemoryScheduleStore139 let recorder: LaunchRecorder140 let engine: SchedulerEngine141142 init(start: String, recorder: LaunchRecorder = LaunchRecorder(), owner: String = "engine-A",143 configuration: SchedulerConfiguration = SchedulerConfiguration(observesSystemWake: false)) {144 let clock = ManualSchedulerClock(now: utc(start))145 self.clock = clock146 store = InMemoryScheduleStore(now: { clock.now() })147 self.recorder = recorder148 engine = SchedulerEngine(store: store, clock: clock, launcher: { await recorder.launch($0, $1) }, owner: owner, configuration: configuration)149 }150151 func secondEngine(owner: String, recorder: LaunchRecorder) -> SchedulerEngine {152 SchedulerEngine(store: store, clock: clock, launcher: { await recorder.launch($0, $1) }, owner: owner,153 configuration: SchedulerConfiguration(observesSystemWake: false))154 }155156 /// One tick, then wait for every launcher to finish.157 @discardableResult func step() async -> TickSummary {158 let summary = await engine.tick()159 await engine.settle()160 return summary161 }162163 func items(_ status: QueueItem.Status? = nil) async -> [QueueItem] {164 (try? await engine.queue(status: status)) ?? []165 }166}167168/// Daily at 09:00 Toronto. In June, 09:00 EDT is 13:00Z.169func dailyNine(_ policy: MissedRunPolicy = .skip, concurrency: ConcurrencyPolicy = .skipIfRunning, createdAt: String) -> Schedule {170 Schedule(id: "daily", workflowID: "wf-report", name: "Daily report", trigger: .cron(expression: "0 9 * * *", timezone: "America/Toronto"),171 missedRunPolicy: policy, concurrency: concurrency, createdAt: utc(createdAt))172}173174@Suite("Scheduler engine: firing and dedup (unit, manual clock, in-memory store)")175struct EngineFiringTests {176 @Test func firesWhenDueAndNotBefore() async throws {177 let h = Harness(start: "2026-06-01T12:00:00Z")178 let saved = try await h.engine.save(dailyNine(createdAt: "2026-06-01T12:00:00Z"))179 #expect(saved.nextFireAt == utc("2026-06-01T13:00:00Z"))180 var summary = await h.step()181 #expect(summary == TickSummary(active: true))182 h.clock.set(utc("2026-06-01T12:59:59Z"))183 #expect(await h.step().enqueued == 0)184 h.clock.set(utc("2026-06-01T13:00:00Z"))185 summary = await h.step()186 #expect(summary.enqueued == 1 && summary.launched == 1 && summary.skipped == 0)187 let items = await h.items()188 #expect(items.count == 1)189 #expect(items[0].status == .succeeded && items[0].attempt == 1 && items[0].runID != nil)190 #expect(items[0].dueAt == utc("2026-06-01T13:00:00Z"))191 #expect(items[0].dedupKey == "daily@\(Int64(utc("2026-06-01T13:00:00Z").timeIntervalSince1970 * 1000))")192 let schedule = try #require(try await h.engine.schedule(id: "daily"))193 #expect(schedule.lastFiredAt == utc("2026-06-01T13:00:00Z") && schedule.nextFireAt == utc("2026-06-02T13:00:00Z"))194 // Ticking again at the same instant changes nothing.195 #expect(await h.step().enqueued == 0)196 #expect(h.recorder.count == 1)197 }198199 @Test func aRestartNeverEnqueuesTheSameInstantTwice() async throws {200 let h = Harness(start: "2026-06-01T12:00:00Z")201 try await h.engine.save(dailyNine(.runAll(max: 5), createdAt: "2026-06-01T12:00:00Z"))202 h.clock.set(utc("2026-06-01T13:00:05Z"))203 await h.step()204 #expect(h.recorder.count == 1)205206 // Simulate a crash right after the enqueue: the schedule's cursor was never saved.207 var stale = try #require(await h.store.schedule(id: "daily"))208 stale.lastEvaluatedAt = utc("2026-06-01T12:00:00Z")209 stale.lastFiredAt = nil210 await h.store.saveSchedule(stale)211212 let afterRestart = LaunchRecorder()213 let restarted = h.secondEngine(owner: "engine-A", recorder: afterRestart)214 let summary = await restarted.tick()215 await restarted.settle()216 #expect(summary.active && summary.enqueued == 0 && summary.launched == 0)217 #expect(afterRestart.count == 0)218 #expect(await h.items().count == 1)219220 // Even long after: the instant is now "missed", the policy says run it, and the key still refuses a duplicate.221 h.clock.set(utc("2026-06-01T15:00:00Z"))222 stale.lastEvaluatedAt = utc("2026-06-01T12:00:00Z")223 await h.store.saveSchedule(stale)224 let later = await restarted.tick()225 await restarted.settle()226 #expect(later.enqueued == 0 && afterRestart.count == 0)227 #expect(await h.items().filter { $0.status != .skipped }.count == 1)228 }229230 @Test func theStoreRefusesASecondItemWithTheSameKey() async throws {231 let store = InMemoryScheduleStore()232 let now = utc("2026-06-01T13:00:00Z")233 let key = QueueItem.dedupKey(scheduleID: "s", instant: now)234 #expect(await store.enqueue(QueueItem(scheduleID: "s", workflowID: "w", dueAt: now, enqueuedAt: now, dedupKey: key)))235 #expect(await store.enqueue(QueueItem(scheduleID: "s", workflowID: "w", dueAt: now, enqueuedAt: now, dedupKey: key)) == false)236 #expect(await store.queueItem(dedupKey: key) != nil)237 #expect(await store.queueItems().count == 1)238 // Atomic claim: one claimer gets it, the next gets nothing.239 let claims = await withTaskGroup(of: QueueItem?.self) { group -> [QueueItem] in240 for _ in 0..<8 { group.addTask { try? await store.claimNext(now: now) } }241 var out: [QueueItem] = []242 for await c in group { if let c { out.append(c) } }243 return out244 }245 #expect(claims.count == 1 && claims[0].status == .running)246 }247248 @Test func onceAndIntervalTriggers() async throws {249 let h = Harness(start: "2026-06-01T12:00:00Z")250 try await h.engine.save(Schedule(id: "once", workflowID: "wf-a", name: "Once", trigger: .once(at: utc("2026-06-01T12:10:00Z")), createdAt: utc("2026-06-01T12:00:00Z")))251 try await h.engine.save(Schedule(id: "every", workflowID: "wf-b", name: "Every 4 min", trigger: .interval(seconds: 240), createdAt: utc("2026-06-01T12:00:00Z")))252 var launchedFor: [String: Int] = [:]253 for minute in 1...20 {254 h.clock.set(utc("2026-06-01T12:00:00Z").addingTimeInterval(Double(minute) * 60))255 await h.step()256 }257 for item in h.recorder.launches { launchedFor[item.scheduleID, default: 0] += 1 }258 #expect(launchedFor == ["once": 1, "every": 5])259 #expect(try await h.engine.schedule(id: "once")?.nextFireAt == nil)260 #expect(try await h.engine.schedule(id: "every")?.nextFireAt == utc("2026-06-01T12:24:00Z"))261 let everyDue = await h.items().filter { $0.scheduleID == "every" }.map(\.dueAt)262 #expect(everyDue == [4, 8, 12, 16, 20].map { utc("2026-06-01T12:00:00Z").addingTimeInterval($0 * 60) })263 }264265 @Test func invalidSchedulesAreRefused() async throws {266 let h = Harness(start: "2026-06-01T12:00:00Z")267 await #expect(throws: SchedulerError.self) {268 try await h.engine.save(Schedule(workflowID: "w", name: "bad", trigger: .cron(expression: "61 * * * *", timezone: nil)))269 }270 await #expect(throws: SchedulerError.self) {271 try await h.engine.save(Schedule(workflowID: "w", name: "bad", trigger: .cron(expression: "0 9 * * *", timezone: "Mars/Olympus")))272 }273 await #expect(throws: SchedulerError.self) {274 try await h.engine.save(Schedule(workflowID: "w", name: "bad", trigger: .interval(seconds: 0.1)))275 }276 await #expect(throws: SchedulerError.self) { try await h.engine.runNow(scheduleID: "nope") }277 #expect(try await h.engine.schedules().isEmpty)278 }279280 @Test func upcomingMergesSchedulesInOrder() async throws {281 let h = Harness(start: "2026-06-01T12:00:00Z")282 try await h.engine.save(dailyNine(createdAt: "2026-06-01T12:00:00Z"))283 try await h.engine.save(Schedule(id: "six-hourly", workflowID: "wf-b", name: "Six-hourly", trigger: .cron(expression: "0 */6 * * *", timezone: "UTC"), createdAt: utc("2026-06-01T12:00:00Z")))284 try await h.engine.save(Schedule(id: "paused", workflowID: "wf-c", name: "Paused", trigger: .interval(seconds: 60), paused: true, createdAt: utc("2026-06-01T12:00:00Z")))285 try await h.engine.save(Schedule(id: "manual", workflowID: "wf-d", name: "Manual", trigger: .manual, createdAt: utc("2026-06-01T12:00:00Z")))286 let upcoming = try await h.engine.upcoming(limit: 5)287 #expect(upcoming.map(\.0.id) == ["daily", "six-hourly", "six-hourly", "six-hourly", "six-hourly"])288 #expect(upcoming.map(\.1) == ["2026-06-01T13:00:00Z", "2026-06-01T18:00:00Z", "2026-06-02T00:00:00Z", "2026-06-02T06:00:00Z", "2026-06-02T12:00:00Z"].map(utc))289 #expect(try await h.engine.upcoming(limit: 0).isEmpty)290 }291}292293@Suite("Scheduler engine: missed runs after a three-day gap (unit, manual clock)")294struct EngineMissedRunTests {295 /// Runs on day 0 at 09:00, then the Mac is off until day 3 at 15:00 Toronto: 09:00 was missed on days 1, 2 and 3.296 private func afterGap(_ policy: MissedRunPolicy, wakeAt: String = "2026-06-04T19:00:00Z") async throws -> Harness {297 let h = Harness(start: "2026-06-01T12:00:00Z")298 try await h.engine.save(dailyNine(policy, concurrency: .queue, createdAt: "2026-06-01T12:00:00Z"))299 h.clock.set(utc("2026-06-01T13:00:00Z"))300 await h.step()301 #expect(h.recorder.count == 1)302 h.clock.set(utc(wakeAt))303 return h304 }305306 @Test func skipRunsNothingAndSaysWhatWasMissed() async throws {307 let h = try await afterGap(.skip)308 let summary = await h.step()309 #expect(summary.enqueued == 0 && summary.launched == 0 && summary.skipped == 1)310 #expect(h.recorder.count == 1)311 let skipped = await h.items(.skipped)312 #expect(skipped.count == 1)313 #expect(skipped[0].reason.contains("3 scheduled firing(s)") && skipped[0].reason.contains("3 not run (policy skip)"))314 #expect(skipped[0].reason.contains("2026-06-02T13:00:00Z") && skipped[0].reason.contains("2026-06-04T13:00:00Z"))315 #expect(skipped[0].runID == nil)316 // It never claims a run for the missed days, and does not repeat the record.317 #expect(await h.items(.succeeded).count == 1)318 #expect(await h.step().skipped == 0)319 #expect(try await h.engine.schedule(id: "daily")?.nextFireAt == utc("2026-06-05T13:00:00Z"))320 }321322 @Test func runOnceRunsTheLatestMissedInstantOnly() async throws {323 let h = try await afterGap(.runOnce)324 let summary = await h.step()325 #expect(summary.enqueued == 1 && summary.launched == 1 && summary.skipped == 1)326 #expect(h.recorder.count == 2)327 let caughtUp = h.recorder.launches[1]328 #expect(caughtUp.dueAt == utc("2026-06-04T13:00:00Z") && caughtUp.reason.hasPrefix("catch-up"))329 #expect(await h.items(.skipped).first?.reason.contains("2 not run (policy runOnce)") == true)330 #expect(await h.step() == TickSummary(active: true))331 }332333 @Test func runOnceDoesNotDoubleAnOnTimeRun() async throws {334 // Wakes 30 s after day 3's 09:00: that one is on time; days 1 and 2 were missed.335 let h = try await afterGap(.runOnce, wakeAt: "2026-06-04T13:00:30Z")336 let summary = await h.step()337 #expect(summary.enqueued == 1 && summary.launched == 1 && summary.skipped == 1)338 #expect(h.recorder.launches.last?.reason.hasPrefix("scheduled for") == true)339 #expect(await h.items(.skipped).first?.reason.contains("2 not run (policy runOnce (covered by the on-time run))") == true)340 }341342 @Test func runAllIsBoundedByMax() async throws {343 let two = try await afterGap(.runAll(max: 2))344 var summary = await two.step()345 #expect(summary.enqueued == 2 && summary.skipped == 1)346 #expect(two.recorder.launches.dropFirst().map(\.dueAt) == ["2026-06-03T13:00:00Z", "2026-06-04T13:00:00Z"].map(utc))347 #expect(await two.items(.skipped).first?.reason.contains("1 not run (policy runAll(max: 2))") == true)348349 let ten = try await afterGap(.runAll(max: 10))350 summary = await ten.step()351 #expect(summary.enqueued == 3 && summary.skipped == 0)352 #expect(ten.recorder.launches.dropFirst().map(\.dueAt) == ["2026-06-02T13:00:00Z", "2026-06-03T13:00:00Z", "2026-06-04T13:00:00Z"].map(utc))353 #expect(await ten.items(.skipped).isEmpty)354 #expect(await ten.items(.succeeded).count == 4)355 }356357 @Test func aBusyEveryMinuteScheduleIsSummarisedNotReplayed() async throws {358 let h = Harness(start: "2026-06-01T12:00:00Z")359 try await h.engine.save(Schedule(id: "minutely", workflowID: "wf", name: "Every minute", trigger: .cron(expression: "* * * * *", timezone: "UTC"),360 missedRunPolicy: .runOnce, concurrency: .queue, createdAt: utc("2026-06-01T12:00:00Z")))361 h.clock.set(utc("2026-06-04T12:00:30Z")) // three days later362 let summary = await h.step()363 // 4320 instants passed; the last two (11:59, 12:00) are within the tolerance and run on time.364 #expect(summary.enqueued == 2 && summary.skipped == 1)365 let reason = try #require(await h.items(.skipped).first?.reason)366 #expect(reason.contains("4318 scheduled firing(s)"), "\(reason)")367 }368}369370@Suite("Scheduler engine: concurrency, retries and control (unit, manual clock)")371struct EngineControlTests {372 private func everyMinute(_ concurrency: ConcurrencyPolicy, maxRetries: Int = 0, backoff: Double = 10) -> Schedule {373 Schedule(id: "s", workflowID: "wf", name: "Every minute", trigger: .interval(seconds: 60), concurrency: concurrency,374 maxRetries: maxRetries, retryBackoffSeconds: backoff, createdAt: utc("2026-06-01T12:00:00Z"))375 }376377 private func advance(_ h: Harness, toMinute m: Double) { h.clock.set(utc("2026-06-01T12:00:00Z").addingTimeInterval(m * 60)) }378379 @Test func skipIfRunningDropsOverlappingFirings() async throws {380 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))381 try await h.engine.save(everyMinute(.skipIfRunning))382 advance(h, toMinute: 1)383 #expect(await h.engine.tick().launched == 1)384 advance(h, toMinute: 2)385 let second = await h.engine.tick()386 #expect(second.enqueued == 1 && second.launched == 0 && second.skipped == 1)387 #expect(await h.items(.running).count == 1)388 #expect(await h.items(.skipped).first?.reason.contains("skipIfRunning") == true)389 h.recorder.release()390 await h.engine.settle()391 advance(h, toMinute: 3)392 await h.step()393 #expect(h.recorder.count == 2)394 #expect(await h.items(.succeeded).count == 2)395 }396397 @Test func queueHoldsOverlappingFiringsUntilTheWorkflowIsFree() async throws {398 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))399 try await h.engine.save(everyMinute(.queue))400 for m in 1...3 {401 advance(h, toMinute: Double(m))402 await h.engine.tick()403 }404 #expect(await eventually { h.recorder.count == 1 })405 #expect(await h.items(.running).count == 1)406 #expect(await h.items(.pending).count == 2)407 // The per-workflow lock also covers another schedule of the same workflow.408 try await h.engine.save(Schedule(id: "other", workflowID: "wf", name: "Other", trigger: .manual, concurrency: .queue, createdAt: utc("2026-06-01T12:00:00Z")))409 h.clock.advance(by: 1)410 try await h.engine.runNow(scheduleID: "other")411 #expect(await h.engine.tick().launched == 0)412 // Completion starts the next one without waiting for a tick, one at a time, oldest first.413 h.recorder.release()414 await h.engine.settle()415 #expect(h.recorder.count == 4)416 #expect(h.recorder.launches.map(\.scheduleID) == ["s", "s", "s", "other"])417 #expect(await h.items(.succeeded).count == 4)418 }419420 @Test func parallelAllowsUpToMaxRuns() async throws {421 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))422 try await h.engine.save(everyMinute(.parallel(max: 2)))423 for m in 1...3 {424 advance(h, toMinute: Double(m))425 await h.engine.tick()426 }427 #expect(await eventually { h.recorder.count == 2 })428 #expect(await h.items(.running).count == 2)429 #expect(await h.items(.pending).count == 1)430 h.recorder.release()431 await h.engine.settle()432 #expect(h.recorder.count == 3)433 }434435 @Test func theGlobalLimitCapsRunsAcrossWorkflows() async throws {436 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true),437 configuration: SchedulerConfiguration(maxConcurrentRuns: 2, observesSystemWake: false))438 for i in 0..<4 {439 try await h.engine.save(Schedule(id: "s\(i)", workflowID: "wf\(i)", name: "S\(i)", trigger: .manual, createdAt: utc("2026-06-01T12:00:00Z")))440 try await h.engine.runNow(scheduleID: "s\(i)")441 }442 #expect(await h.engine.tick().launched == 2)443 #expect(await h.items(.pending).count == 2)444 h.recorder.release()445 await h.engine.settle()446 #expect(h.recorder.count == 4)447 }448449 @Test func retriesOnlyRetryableFailuresWithDoublingBackoff() async throws {450 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .failed(runID: "r", error: "network down", retryable: true) }))451 try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Once", trigger: .once(at: utc("2026-06-01T12:01:00Z")),452 maxRetries: 2, retryBackoffSeconds: 10, createdAt: utc("2026-06-01T12:00:00Z")))453 advance(h, toMinute: 1)454 await h.step()455 var item = try #require(await h.items().first)456 #expect(item.status == .pending && item.attempt == 2 && item.lastError == "network down")457 #expect(item.dueAt == utc("2026-06-01T12:01:10Z"))458 // Not before the backoff has elapsed.459 h.clock.advance(by: 9)460 #expect(await h.step().launched == 0)461 h.clock.advance(by: 1)462 #expect(await h.step().launched == 1)463 item = try #require(await h.items().first)464 #expect(item.status == .pending && item.attempt == 3 && item.dueAt == utc("2026-06-01T12:01:30Z"))465 h.clock.advance(by: 20)466 await h.step()467 item = try #require(await h.items().first)468 #expect(item.status == .failed && item.attempt == 3)469 #expect(h.recorder.count == 3)470 h.clock.advance(by: 3600)471 await h.step()472 #expect(h.recorder.count == 3)473 }474475 @Test func nonRetryableFailuresAndZeroRetriesAreFinal() async throws {476 let notRetryable = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .failed(runID: nil, error: "mail already sent", retryable: false) }))477 try await notRetryable.engine.save(everyMinute(.queue, maxRetries: 5))478 advance(notRetryable, toMinute: 1)479 await notRetryable.step()480 #expect(await notRetryable.items(.failed).count == 1)481 #expect(notRetryable.recorder.count == 1)482483 let noRetries = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .failed(runID: nil, error: "boom", retryable: true) }))484 try await noRetries.engine.save(everyMinute(.queue, maxRetries: 0))485 advance(noRetries, toMinute: 1)486 await noRetries.step()487 #expect(await noRetries.items(.failed).first?.lastError == "boom")488 #expect(noRetries.recorder.count == 1)489 }490491 @Test func backoffIsCapped() async throws {492 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .failed(runID: nil, error: "x", retryable: true) }),493 configuration: SchedulerConfiguration(maxBackoffSeconds: 25, observesSystemWake: false))494 try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Manual", trigger: .manual, maxRetries: 4, retryBackoffSeconds: 10, createdAt: utc("2026-06-01T12:00:00Z")))495 try await h.engine.runNow(scheduleID: "s")496 var delays: [Double] = []497 for _ in 0..<4 {498 await h.step()499 let item = try #require(await h.items().first)500 delays.append(item.dueAt.timeIntervalSince(h.clock.now()))501 h.clock.set(item.dueAt)502 }503 #expect(delays == [10, 20, 25, 25])504 }505506 @Test func pauseHoldsAndResumeDoesNotCatchUp() async throws {507 let h = Harness(start: "2026-06-01T12:00:00Z")508 try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Every minute", trigger: .interval(seconds: 60),509 missedRunPolicy: .runAll(max: 100), createdAt: utc("2026-06-01T12:00:00Z")))510 try await h.engine.pause(scheduleID: "s")511 #expect(try await h.engine.schedule(id: "s")?.nextFireAt == nil)512 advance(h, toMinute: 30)513 #expect(await h.step() == TickSummary(active: true))514 try await h.engine.resume(scheduleID: "s")515 #expect(try await h.engine.schedule(id: "s")?.nextFireAt == utc("2026-06-01T12:31:00Z"))516 #expect(await h.step().enqueued == 0)517 advance(h, toMinute: 31)518 #expect(await h.step().enqueued == 1)519 #expect(h.recorder.count == 1)520 }521522 @Test func runNowWorksWhilePausedAndIsNotDeduplicated() async throws {523 let h = Harness(start: "2026-06-01T12:00:00Z")524 try await h.engine.save(everyMinute(.queue))525 try await h.engine.pause(scheduleID: "s")526 let first = try await h.engine.runNow(scheduleID: "s")527 let second = try await h.engine.runNow(scheduleID: "s")528 #expect(first.dedupKey != second.dedupKey && first.reason == "manual: run now")529 await h.step()530 #expect(h.recorder.count == 2)531 #expect(await h.items(.succeeded).count == 2)532 #expect(try await h.engine.schedule(id: "s")?.lastFiredAt == utc("2026-06-01T12:00:00Z"))533 }534535 @Test func cancelPendingRunningAndFinishedItems() async throws {536 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))537 try await h.engine.save(everyMinute(.queue))538 advance(h, toMinute: 1)539 await h.engine.tick()540 advance(h, toMinute: 2)541 await h.engine.tick()542 let running = try #require(await h.items(.running).first)543 let pending = try #require(await h.items(.pending).first)544 try await h.engine.cancel(queueItemID: pending.id)545 try await h.engine.cancel(queueItemID: running.id)546 #expect(await h.items(.canceled).count == 2)547 // The launcher's late "succeeded" does not resurrect the canceled run.548 h.recorder.release()549 await h.engine.settle()550 #expect(await h.items(.canceled).count == 2)551 #expect(await h.items(.succeeded).isEmpty)552 #expect(h.recorder.count == 1)553 // Canceling a finished item is a no-op; an unknown one is an error.554 try await h.engine.cancel(queueItemID: running.id)555 await #expect(throws: SchedulerError.self) { try await h.engine.cancel(queueItemID: "nope") }556 }557558 @Test func disableCancelsPendingAndStopsFiring() async throws {559 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(held: true))560 try await h.engine.save(everyMinute(.queue))561 for m in 1...3 { advance(h, toMinute: Double(m)); await h.engine.tick() }562 try await h.engine.disable(scheduleID: "s")563 #expect(await h.items(.canceled).map(\.reason) == ["schedule disabled", "schedule disabled"])564 h.recorder.release()565 await h.engine.settle()566 advance(h, toMinute: 10)567 #expect(await h.step() == TickSummary(active: true))568 #expect(h.recorder.count == 1)569 try await h.engine.enable(scheduleID: "s")570 advance(h, toMinute: 11)571 #expect(await h.step().enqueued == 1)572 // Removing the schedule cancels what is still pending.573 try await h.engine.pause(scheduleID: "s")574 _ = try await h.engine.fire(scheduleID: "s", reason: "ignored while paused")575 try await h.engine.resume(scheduleID: "s")576 advance(h, toMinute: 12)577 try await h.engine.remove(scheduleID: "s")578 #expect(await h.step().launched == 0)579 }580581 @Test func eventFiringRespectsEnabledPausedAndDedupKeys() async throws {582 let h = Harness(start: "2026-06-01T12:00:00Z")583 try await h.engine.save(Schedule(id: "login", workflowID: "wf", name: "At login", trigger: .login, concurrency: .queue, createdAt: utc("2026-06-01T12:00:00Z")))584 #expect(try await h.engine.fire(scheduleID: "login", reason: "login", dedupKey: "login@login-1") != nil)585 #expect(try await h.engine.fire(scheduleID: "login", reason: "login", dedupKey: "login@login-1") == nil)586 #expect(try await h.engine.fire(scheduleID: "missing", reason: "x") == nil)587 try await h.engine.pause(scheduleID: "login")588 #expect(try await h.engine.fire(scheduleID: "login", reason: "x") == nil)589 try await h.engine.resume(scheduleID: "login")590 try await h.engine.disable(scheduleID: "login")591 #expect(try await h.engine.fire(scheduleID: "login", reason: "x") == nil)592 #expect(await h.items().count == 1)593 }594}595596@Suite("Scheduler engine: approvals, lease and run loop (unit, manual clock)")597struct EngineApprovalLeaseTests {598 @Test func waitingApprovalItemsStayParkedAndVisible() async throws {599 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .waitingApproval(runID: "run-42") }))600 try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Every minute", trigger: .interval(seconds: 60), createdAt: utc("2026-06-01T12:00:00Z")))601 h.clock.advance(by: 60)602 await h.step()603 let parked = try #require(await h.items(.waitingApproval).first)604 #expect(parked.runID == "run-42" && parked.reason.contains("waiting for approval"))605 // Hours of ticking never approve, relaunch or drop it; the workflow stays locked.606 for _ in 0..<5 {607 h.clock.advance(by: 3600)608 await h.step()609 }610 #expect(h.recorder.count == 1)611 #expect(await h.items(.waitingApproval).map(\.id) == [parked.id])612 #expect(await h.items(.succeeded).isEmpty)613 #expect(await h.items(.skipped).contains { $0.reason.contains("skipIfRunning") })614 // Only an explicit resolution ends it.615 await #expect(throws: SchedulerError.self) { try await h.engine.resolveApproval(queueItemID: "nope", result: .succeeded(runID: nil)) }616 try await h.engine.resolveApproval(queueItemID: parked.id, result: .succeeded(runID: "run-42"))617 #expect(await h.store.queueItem(id: parked.id)?.status == .succeeded)618 await #expect(throws: SchedulerError.self) { try await h.engine.resolveApproval(queueItemID: parked.id, result: .succeeded(runID: nil)) }619 }620621 @Test func aParkedItemCanBeCanceled() async throws {622 let h = Harness(start: "2026-06-01T12:00:00Z", recorder: LaunchRecorder(answer: { _ in .waitingApproval(runID: nil) }))623 try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Manual", trigger: .manual, createdAt: utc("2026-06-01T12:00:00Z")))624 let item = try await h.engine.runNow(scheduleID: "s")625 await h.step()626 try await h.engine.cancel(queueItemID: item.id)627 #expect(await h.store.queueItem(id: item.id)?.status == .canceled)628 }629630 @Test func aSecondEngineStaysPassiveThenTakesOverWhenTheLeaseExpires() async throws {631 let h = Harness(start: "2026-06-01T12:00:00Z")632 let recorderB = LaunchRecorder()633 let b = h.secondEngine(owner: "engine-B", recorder: recorderB)634 try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Every minute", trigger: .interval(seconds: 60),635 missedRunPolicy: .runAll(max: 10), createdAt: utc("2026-06-01T12:00:00Z")))636 #expect(await h.engine.tick().active)637 #expect(await b.tick().active == false)638 #expect(await b.isActive == false)639 #expect(await h.store.leaseOwner(name: "orix.scheduler") == "engine-A")640641 // A keeps renewing: B stays passive even when something is due.642 h.clock.advance(by: 20)643 await h.engine.tick()644 h.clock.advance(by: 20)645 await h.engine.tick()646 h.clock.advance(by: 20) // 12:01:00, due647 let passive = await b.tick()648 #expect(passive == TickSummary(active: false))649 #expect(recorderB.count == 0)650 await h.step()651 #expect(h.recorder.count == 1)652653 // A dies. Its lease (30 s) runs out; B takes over and handles what A never did.654 h.clock.advance(by: 29)655 #expect(await b.tick().active == false)656 h.clock.advance(by: 32) // 12:02:01657 let takeover = await b.tick()658 await b.settle()659 #expect(takeover.active && takeover.enqueued == 1)660 #expect(recorderB.count == 1 && h.recorder.count == 1)661 #expect(await h.store.leaseOwner(name: "orix.scheduler") == "engine-B")662 // A comes back: it is the passive one now, and no instant ran twice.663 #expect(await h.engine.tick().active == false)664 #expect(await h.engine.isActive == false)665 let due = await h.items(.succeeded).map(\.dueAt)666 #expect(due == [utc("2026-06-01T12:01:00Z"), utc("2026-06-01T12:02:00Z")])667 }668669 @Test func stopReleasesTheLeaseImmediately() async throws {670 let h = Harness(start: "2026-06-01T12:00:00Z")671 let b = h.secondEngine(owner: "engine-B", recorder: LaunchRecorder())672 await h.engine.tick()673 #expect(await b.tick().active == false)674 await h.engine.stop()675 #expect(await b.tick().active)676 }677678 @Test func runsInterruptedByARestartAreMarkedFailedAndNotRelaunched() async throws {679 let h = Harness(start: "2026-06-01T12:00:00Z")680 try await h.engine.save(Schedule(id: "s", workflowID: "wf", name: "Manual", trigger: .manual, maxRetries: 3, createdAt: utc("2026-06-01T12:00:00Z")))681 let now = h.clock.now()682 _ = await h.store.enqueue(QueueItem(id: "orphan", scheduleID: "s", workflowID: "wf", dueAt: now, enqueuedAt: now, status: .running, dedupKey: "s@orphan"))683 let summary = await h.step()684 #expect(summary.launched == 0 && h.recorder.count == 0)685 let orphan = try #require(await h.store.queueItem(id: "orphan"))686 #expect(orphan.status == .failed && orphan.lastError?.contains("interrupted") == true)687 }688689 @Test func theRunLoopSleepsUntilTheNextInstantAndWakesOnChanges() async throws {690 let h = Harness(start: "2026-06-01T12:00:00Z", configuration: SchedulerConfiguration(heartbeatSeconds: 3600, observesSystemWake: false))691 try await h.engine.save(dailyNine(createdAt: "2026-06-01T12:00:00Z"))692 let loop = Task { await h.engine.run() }693 defer { loop.cancel() }694695 func eventually(_ what: String, _ condition: @Sendable () async -> Bool) async {696 for _ in 0..<400 {697 if await condition() { return }698 try? await Task.sleep(nanoseconds: 10_000_000)699 }700 Issue.record("timed out waiting for: \(what)")701 }702 // The loop parks on the clock; nothing fires early.703 await eventually("loop asleep") { h.clock.sleeperCount == 1 }704 #expect(h.recorder.count == 0)705 // Reaching the instant wakes it without any poke.706 h.clock.set(utc("2026-06-01T13:00:00Z"))707 await eventually("scheduled run") { await h.items(.succeeded).count == 1 }708 // A change made through the engine wakes it at once, with the clock standing still.709 await eventually("loop asleep again") { h.clock.sleeperCount == 1 }710 try await h.engine.runNow(scheduleID: "daily")711 await eventually("manual run") { await h.items(.succeeded).count == 2 }712 // So does an outside store change announced with storeDidChange().713 await eventually("loop asleep again") { h.clock.sleeperCount == 1 }714 let now = h.clock.now()715 _ = await h.store.enqueue(QueueItem(scheduleID: "daily", workflowID: "wf-report", dueAt: now, enqueuedAt: now, dedupKey: "daily@external"))716 h.engine.storeDidChange()717 await eventually("external item") { await h.items(.succeeded).count == 3 }718 await h.engine.stop()719 await eventually("loop ended") { h.clock.sleeperCount == 0 }720 _ = await loop.value721 }722}723724@Suite("Scheduler models (unit)")725struct SchedulerModelTests {726 @Test func modelsRoundTripThroughJSON() throws {727 let triggers: [TriggerSpec] = [728 .cron(expression: "0 9 * * MON", timezone: "America/Toronto"), .cron(expression: "@daily", timezone: nil),729 .once(at: utc("2026-06-01T12:00:00Z")), .interval(seconds: 90),730 .fileChange(path: "/tmp/in", recursive: true, glob: "*.pdf", debounceSeconds: 2), .login,731 .appLaunch(bundleID: "com.apple.Safari"), .powerConnected, .networkAvailable, .manual,732 ]733 let encoder = JSONEncoder(), decoder = JSONDecoder()734 for (i, trigger) in triggers.enumerated() {735 let schedule = Schedule(workflowID: "wf", name: "S\(i)", trigger: trigger, missedRunPolicy: i % 2 == 0 ? .runAll(max: 3) : .runOnce,736 concurrency: i % 2 == 0 ? .parallel(max: 2) : .queue, maxRetries: 2, lastFiredAt: utc("2026-06-01T12:00:00Z"))737 #expect(try decoder.decode(Schedule.self, from: encoder.encode(schedule)) == schedule)738 #expect(schedule.problems().isEmpty)739 }740 let item = QueueItem(scheduleID: "s", workflowID: "wf", dueAt: utc("2026-06-01T12:00:00Z"), enqueuedAt: utc("2026-06-01T12:00:01Z"),741 attempt: 2, status: .waitingApproval, dedupKey: "s@1", reason: "r", runID: "run", lastError: "e")742 #expect(try decoder.decode(QueueItem.self, from: encoder.encode(item)) == item)743 #expect(Set(QueueItem.Status.allCases.map(\.rawValue)) == ["pending", "running", "succeeded", "failed", "skipped", "canceled", "waitingApproval"])744 #expect(QueueItem.Status.allCases.filter { !$0.isTerminal } == [.pending, .running, .waitingApproval])745 }746747 @Test func triggerMathForIntervalAndOnce() {748 let anchor = utc("2026-06-01T12:00:00Z")749 let every = TriggerSpec.interval(seconds: 90)750 #expect(every.nextFireDate(after: anchor, anchor: anchor) == anchor.addingTimeInterval(90))751 #expect(every.nextFireDate(after: anchor.addingTimeInterval(90), anchor: anchor) == anchor.addingTimeInterval(180))752 #expect(every.nextFireDate(after: anchor.addingTimeInterval(-500), anchor: anchor) == anchor.addingTimeInterval(90))753 #expect(every.latestFireDate(after: anchor, atOrBefore: anchor.addingTimeInterval(400), anchor: anchor) == anchor.addingTimeInterval(360))754 #expect(every.latestFireDate(after: anchor, atOrBefore: anchor.addingTimeInterval(89), anchor: anchor) == nil)755 let once = TriggerSpec.once(at: anchor.addingTimeInterval(10))756 #expect(once.nextFireDate(after: anchor, anchor: anchor) == anchor.addingTimeInterval(10))757 #expect(once.nextFireDate(after: anchor.addingTimeInterval(10), anchor: anchor) == nil)758 #expect(once.latestFireDate(after: anchor, atOrBefore: anchor.addingTimeInterval(60), anchor: anchor) == anchor.addingTimeInterval(10))759 #expect(TriggerSpec.manual.nextFireDate(after: anchor, anchor: anchor) == nil && !TriggerSpec.login.isTimeBased)760 #expect(TriggerSpec.cron(expression: "bad", timezone: nil).problems().count == 1)761 #expect(Schedule(workflowID: "", name: "x", trigger: .manual, missedRunPolicy: .runAll(max: 0), concurrency: .parallel(max: 0), maxRetries: -1).problems().count == 4)762 #expect(ConcurrencyPolicy.parallel(max: 3).capacity == 3 && ConcurrencyPolicy.queue.capacity == 1)763 }764765 @Test func manualClockSleepersWakeInOrderAndHonourCancellation() async throws {766 let clock = ManualSchedulerClock(now: utc("2026-06-01T12:00:00Z"))767 try await clock.sleep(until: utc("2026-06-01T11:00:00Z")) // already past768 let sleeper = Task { try await clock.sleep(until: utc("2026-06-01T12:00:10Z")); return clock.now() }769 while clock.sleeperCount == 0 { await Task.yield() }770 clock.advance(by: 5)771 #expect(clock.sleeperCount == 1)772 clock.advance(by: 5)773 #expect(try await sleeper.value == utc("2026-06-01T12:00:10Z"))774 let canceled = Task { try await clock.sleep(until: utc("2027-01-01T00:00:00Z")) }775 while clock.sleeperCount == 0 { await Task.yield() }776 canceled.cancel()777 await #expect(throws: CancellationError.self) { try await canceled.value }778 #expect(clock.sleeperCount == 0)779 }780}781