Repository Public License
1// ORIX Convert — Packages/ORIXConvertCore/Sources/ConvertCore/Jobs/ConversionQueue.swift2// Description: The conversion queue: an actor that owns every job, schedules them by priority within the resource3// limits (concurrency, heavy class, thermal and power state), and publishes events and snapshots.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation67/// The job queue.8///9/// **Pause semantics.** `pause()` stops new jobs from starting; jobs already running continue to their end.10/// `pauseJob` holds back a job that has not started. A running job cannot be paused: engines offer no cooperative11/// suspension point, so the queue does not pretend to — cancel it and retry later instead.12///13/// **Cancellation.** `cancel` cancels the job's task. Engines check `context.checkCancellation()` in their loops14/// and `ProcessRunner` terminates helper processes; the job becomes `cancelled` when the engine returns.15///16/// **Errors.** A failed job never affects the other jobs of its batch.17public actor ConversionQueue {18 public nonisolated let registry: FormatRegistry19 private var configuration: QueueConfiguration20 private let history: ConversionHistoryStore?21 private let workDirectories: WorkDirectoryManager22 private let monitor: any ResourceMonitoring2324 private struct Record {25 var job: ConversionJob26 var task: Task<Void, Never>?27 var held: HeldOutput?28 var policy: ConflictPolicy29 var allowReplacingSource: Bool30 var waiters: [CheckedContinuation<ConversionJob, Never>] = []31 }3233 private var records: [UUID: Record] = [:]34 private var order: [UUID] = []35 private var paused = false36 private var subscribers: [UUID: AsyncStream<QueueEvent>.Continuation] = [:]37 private var idleWaiters: [CheckedContinuation<Void, Never>] = []38 private var lastStatus: QueueStatus?39 private var wasIdle = true40 private var pendingHistory: [UUID: Int] = [:]41 private let observers = ObserverBag()4243 /// - Parameters:44 /// - history: when given, every job that ends (finished, failed, cancelled, skipped) is recorded.45 /// - workDirectories: where engines write. Directories left by dead processes are swept now.46 /// - monitor: thermal / power state source (the system by default).47 public init(registry: FormatRegistry, configuration: QueueConfiguration = QueueConfiguration(), history: ConversionHistoryStore? = nil,48 workDirectories: WorkDirectoryManager = WorkDirectoryManager(), monitor: any ResourceMonitoring = SystemResourceMonitor()) {49 self.registry = registry50 self.configuration = configuration51 self.history = history52 self.workDirectories = workDirectories53 self.monitor = monitor54 workDirectories.sweepStale()55 let center = NotificationCenter.default56 for name in [ProcessInfo.thermalStateDidChangeNotification, Notification.Name.NSProcessInfoPowerStateDidChange] {57 observers.add(center.addObserver(forName: name, object: nil, queue: nil) { [weak self] _ in58 Task { await self?.resourceStateChanged() }59 })60 }61 }6263 // MARK: Enqueueing6465 /// Adds a job and returns its identifier. The job starts as soon as a slot is free.66 @discardableResult67 public func enqueue(_ request: ConversionJobRequest) -> UUID {68 let id = UUID()69 let job = ConversionJob(70 id: id, request: request, state: .queued, priority: request.priority, sourceFormat: request.sourceFormat, detection: nil,71 plan: nil, progress: JobProgress(), estimatedSecondsRemaining: nil,72 log: [JobLogLine(level: .info, message: "Queued.")], warnings: [], checks: [], outputs: [], facts: [:], failure: nil,73 conflict: nil, createdAt: Date(), startedAt: nil, finishedAt: nil, attempt: 1, isHeavy: isHeavy(request), bytesBefore: nil,74 bytesAfter: nil, cancelRequested: false)75 records[id] = Record(job: job, policy: request.destination.conflictPolicy, allowReplacingSource: request.destination.allowReplacingSource)76 order.append(id)77 emit(.jobAdded(job))78 pump()79 return id80 }8182 /// Adds several jobs as one batch: they share a `batchID` and get counters 1…n (unless already set).83 @discardableResult84 public func enqueue(batch requests: [ConversionJobRequest]) -> [UUID] {85 let batchID = UUID()86 return requests.enumerated().map { index, request in87 var request = request88 request.batchID = request.batchID ?? batchID89 if request.counter <= 1 { request.counter = index + 1 }90 return enqueue(request)91 }92 }9394 // MARK: Control9596 /// Stops starting new jobs. Running jobs continue.97 public func pause() {98 guard !paused else { return }99 paused = true100 publishStatus()101 }102103 public func resume() {104 guard paused else { return }105 paused = false106 pump()107 }108109 public var isPaused: Bool { paused }110111 /// Holds back a job that has not started. Returns `false` for a job that is running (see the type's notes),112 /// settled or unknown.113 @discardableResult114 public func pauseJob(_ id: UUID) -> Bool {115 guard records[id]?.job.state == .queued else { return false }116 transition(id, to: .paused, log: "Paused before start.")117 pump()118 return true119 }120121 @discardableResult122 public func resumeJob(_ id: UUID) -> Bool {123 guard records[id]?.job.state == .paused else { return false }124 transition(id, to: .queued, log: "Resumed.")125 pump()126 return true127 }128129 /// Cancels a job. A waiting job is cancelled at once; a running job is cancelled cooperatively.130 @discardableResult131 public func cancel(_ id: UUID) -> Bool {132 guard var record = records[id] else { return false }133 switch record.job.state {134 case .queued, .paused, .awaitingDecision:135 if let held = record.held { workDirectories.remove(held.workDirectory) }136 record.held = nil137 record.job.conflict = nil138 records[id] = record139 settle(id, state: .cancelled, log: "Cancelled.")140 return true141 case .preparing, .running, .validating:142 record.job.cancelRequested = true143 record.job.log.append(JobLogLine(level: .info, message: "Cancellation requested."))144 records[id] = record145 record.task?.cancel()146 emit(.jobChanged(record.job))147 return true148 case .finished, .failed, .cancelled, .skipped:149 return false150 }151 }152153 /// Cancels every job that is not settled.154 public func cancelAll() {155 for id in order { cancel(id) }156 }157158 /// Runs a failed, cancelled or skipped job again (same identifier, `attempt` incremented).159 @discardableResult160 public func retry(_ id: UUID) -> Bool {161 guard var record = records[id], [.failed, .cancelled, .skipped].contains(record.job.state), record.task == nil else { return false }162 record.job.attempt += 1163 record.job.state = .queued164 record.job.failure = nil165 record.job.conflict = nil166 record.job.checks = []167 record.job.warnings = []168 record.job.outputs = []169 record.job.facts = [:]170 record.job.progress = JobProgress()171 record.job.estimatedSecondsRemaining = nil172 record.job.startedAt = nil173 record.job.finishedAt = nil174 record.job.cancelRequested = false175 record.policy = record.job.request.destination.conflictPolicy176 record.allowReplacingSource = record.job.request.destination.allowReplacingSource177 record.job.log.append(JobLogLine(level: .info, message: "Retry (attempt \(record.job.attempt))."))178 records[id] = record179 emit(.jobChanged(record.job))180 pump()181 return true182 }183184 /// Changes the priority of a job that has not started.185 @discardableResult186 public func setPriority(_ priority: JobPriority, for id: UUID) -> Bool {187 guard var record = records[id], record.job.state == .queued || record.job.state == .paused else { return false }188 record.job.priority = priority189 records[id] = record190 emit(.jobChanged(record.job))191 return true192 }193194 /// Moves a job to `index` in the display order (which is the start order within a priority).195 @discardableResult196 public func move(_ id: UUID, to index: Int) -> Bool {197 guard let current = order.firstIndex(of: id) else { return false }198 order.remove(at: current)199 order.insert(id, at: min(max(0, index), order.count))200 if let job = records[id]?.job { emit(.jobChanged(job)) }201 return true202 }203204 /// Removes a job from the list. Running jobs must be cancelled first.205 @discardableResult206 public func remove(_ id: UUID) -> Bool {207 guard let record = records[id], !record.job.state.isActive, record.task == nil else { return false }208 if let held = record.held { workDirectories.remove(held.workDirectory) }209 for waiter in record.waiters { waiter.resume(returning: record.job) }210 records[id] = nil211 order.removeAll { $0 == id }212 emit(.jobRemoved(id))213 pump()214 return true215 }216217 /// Removes every job in a terminal state. Returns how many were removed.218 @discardableResult219 public func clearSettled() -> Int {220 let ids = order.filter { records[$0]?.job.state.isTerminal == true && records[$0]?.task == nil }221 ids.forEach { remove($0) }222 return ids.count223 }224225 /// Decides a job in `awaitingDecision`.226 ///227 /// - `rename` / `replace`: writes the output (or starts the conversion when the conflict was found beforehand)228 /// under that policy. Replacing the job's own source additionally needs `allowReplacingSource`.229 /// - `skip`: keeps the existing file; the job ends `skipped`.230 /// - `askUser` is not a decision and returns `false`.231 @discardableResult232 public func resolveConflict(_ id: UUID, policy: ConflictPolicy, allowReplacingSource: Bool = false) -> Bool {233 guard var record = records[id], record.job.state == .awaitingDecision, policy != .askUser else { return false }234 record.policy = policy235 record.allowReplacingSource = allowReplacingSource || record.allowReplacingSource236 let conflict = record.job.conflict237 record.job.conflict = nil238 guard let held = record.held else {239 // Conflict found before converting.240 records[id] = record241 if policy == .skip {242 settle(id, state: .skipped, log: "Skipped: the existing file was kept.")243 } else {244 transition(id, to: .queued, log: "Decision: \(policy.rawValue).")245 pump()246 }247 return true248 }249 record.held = nil250 records[id] = record251 if policy == .skip {252 workDirectories.remove(held.workDirectory)253 settle(id, state: record.job.outputs.contains { $0.outcome != .skipped } ? .finished : .skipped, log: "Skipped: the existing file was kept.")254 return true255 }256 do {257 let committed = try SafeWriter.commit(held.items, names: held.names, to: held.directory, policy: policy, sources: record.job.request.sources,258 allowReplacingSource: record.allowReplacingSource)259 workDirectories.remove(held.workDirectory)260 records[id]?.job.outputs += committed261 let written = (records[id]?.job.outputs ?? []).filter { $0.outcome != .skipped }262 records[id]?.job.bytesAfter = written.compactMap { FileSizer.size(of: $0.url) }.reduce(Int64(0), +)263 settle(id, state: .finished, log: "Decision: \(policy.rawValue). Wrote \(committed.map(\.url.lastPathComponent).joined(separator: ", ")).")264 } catch ConversionError.destinationExists {265 // Still not allowed (e.g. replacing the source without permission): keep waiting.266 records[id]?.held = held267 records[id]?.job.conflict = conflict268 return false269 } catch {270 workDirectories.remove(held.workDirectory)271 records[id]?.job.failure = (error as? ConversionError) ?? .writeFailed(error.localizedDescription)272 settle(id, state: .failed, log: "Writing failed: \(error.localizedDescription)")273 }274 return true275 }276277 /// Replaces the configuration (limits apply to the next scheduling decision).278 public func updateConfiguration(_ configuration: QueueConfiguration) {279 self.configuration = configuration280 pump()281 }282283 public var currentConfiguration: QueueConfiguration { configuration }284285 /// Re-reads the thermal and power state and reschedules. Called automatically on thermal and Low Power Mode286 /// notifications; the battery state is re-read at every scheduling decision.287 public func resourceStateChanged() {288 pump()289 }290291 // MARK: Observation292293 public func snapshot() -> QueueSnapshot {294 QueueSnapshot(jobs: order.compactMap { records[$0]?.job }, status: status())295 }296297 public func job(_ id: UUID) -> ConversionJob? { records[id]?.job }298299 /// A stream of everything that happens in the queue, from now on. Each call returns an independent stream;300 /// cancel the consuming task (or drop the stream) to unsubscribe. A slow consumer loses the oldest events301 /// beyond 4096 buffered ones — use `snapshot()` to resynchronise.302 public func events() -> AsyncStream<QueueEvent> {303 let token = UUID()304 let (stream, continuation) = AsyncStream<QueueEvent>.makeStream(bufferingPolicy: .bufferingNewest(4096))305 subscribers[token] = continuation306 continuation.onTermination = { [weak self] _ in307 Task { await self?.unsubscribe(token) }308 }309 return stream310 }311312 /// Waits until the job is settled (finished, failed, cancelled, skipped, or awaiting a decision) and returns its313 /// snapshot; `nil` for an unknown job.314 public func waitForJob(_ id: UUID) async -> ConversionJob? {315 guard let record = records[id] else { return nil }316 if record.job.state.isSettled, record.task == nil, pendingHistory[id] == nil { return record.job }317 return await withCheckedContinuation { continuation in318 records[id]?.waiters.append(continuation)319 }320 }321322 /// Waits until nothing is running and nothing can start (queue empty of startable jobs, or paused).323 public func waitUntilIdle() async {324 if isIdle { return }325 await withCheckedContinuation { idleWaiters.append($0) }326 }327328 // MARK: Scheduling329330 private var isIdle: Bool {331 let jobs = records.values332 if !pendingHistory.isEmpty || jobs.contains(where: { $0.job.state.isActive || $0.task != nil }) { return false }333 return paused || !jobs.contains { $0.job.state == .queued }334 }335336 private func status() -> QueueStatus {337 let state = monitor.currentState()338 let jobs = records.values.map(\.job)339 return QueueStatus(isPaused: paused, limits: configuration.limits(for: state), resources: state,340 queued: jobs.filter { $0.state == .queued || $0.state == .paused }.count,341 active: jobs.filter(\.state.isActive).count, settled: jobs.filter(\.state.isSettled).count)342 }343344 private func publishStatus() {345 let current = status()346 if current != lastStatus {347 lastStatus = current348 emit(.queueChanged(current))349 }350 let idle = isIdle351 if idle {352 if !wasIdle { emit(.idle) }353 let waiters = idleWaiters354 idleWaiters = []355 waiters.forEach { $0.resume() }356 }357 wasIdle = idle358 }359360 /// Starts as many queued jobs as the limits allow: highest priority first, then display order. A heavy job that361 /// cannot start because the heavy slots are taken does not block lighter jobs behind it.362 private func pump() {363 defer { publishStatus() }364 guard !paused else { return }365 let limits = configuration.limits(for: monitor.currentState())366 var total = records.values.filter { $0.job.state.isActive }.count367 var heavy = records.values.filter { $0.job.state.isActive && $0.job.isHeavy }.count368 let candidates = order.enumerated()369 .compactMap { index, id in records[id].map { (index, $0.job) } }370 .filter { $0.1.state == .queued && records[$0.1.id]?.task == nil }371 .sorted { ($1.1.priority.rawValue, $0.0) < ($0.1.priority.rawValue, $1.0) }372 for (_, job) in candidates {373 guard total < limits.total else { break }374 if job.isHeavy {375 guard heavy < limits.heavy else { continue }376 heavy += 1377 }378 total += 1379 launch(job.id)380 }381 }382383 private func isHeavy(_ request: ConversionJobRequest) -> Bool {384 if let engine = request.engineID, configuration.heavyEngineIDs.contains(engine) { return true }385 let guessed = request.sourceFormat ?? request.sources.first.flatMap { FormatCatalog.formats(forExtension: $0.pathExtension).first?.id }386 if [guessed, request.destinationFormat].contains(where: { $0.flatMap(FormatCatalog.format)?.category == .video }) { return true }387 if let guessed, let plan = try? ConversionPlanner(registry: registry).plan(from: guessed, to: request.destinationFormat, operation: request.operation,388 engine: request.engineID, allowChaining: request.allowChaining),389 !configuration.heavyEngineIDs.isDisjoint(with: plan.engines) { return true }390 let size = request.sources.reduce(Int64(0)) { total, url in391 total + (((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? NSNumber)?.int64Value ?? 0)392 }393 return size >= configuration.heavyFileSize394 }395396 private func launch(_ id: UUID) {397 guard var record = records[id] else { return }398 record.job.state = .preparing399 record.job.startedAt = Date()400 record.job.log.append(JobLogLine(level: .info, message: "Started."))401 let attempt = record.job.attempt402 let (stream, continuation) = AsyncStream<JobSignal>.makeStream(bufferingPolicy: .bufferingNewest(8192))403 let runner = JobRunner(id: id, request: record.job.request, policy: record.policy, allowReplacingSource: record.allowReplacingSource,404 registry: registry, workDirectories: workDirectories, signal: { continuation.yield($0) })405 record.task = Task {406 let forwarder = Task {407 for await signal in stream { self.apply(signal, to: id, attempt: attempt) }408 }409 let report = await runner.run()410 continuation.finish()411 await forwarder.value412 self.complete(id, attempt: attempt, report: report)413 }414 records[id] = record415 emit(.jobChanged(record.job))416 }417418 private func apply(_ signal: JobSignal, to id: UUID, attempt: Int) {419 guard var record = records[id], record.job.attempt == attempt, record.task != nil else { return }420 switch signal {421 case .state(let state):422 record.job.state = state423 records[id] = record424 emit(.jobChanged(record.job))425 case .prepared(let sourceFormat, let detection, let plan, let bytesBefore):426 record.job.sourceFormat = sourceFormat427 record.job.detection = detection428 record.job.plan = plan429 record.job.bytesBefore = bytesBefore430 record.job.progress = JobProgress(stepIndex: 0, stepCount: plan.steps.count)431 records[id] = record432 emit(.jobChanged(record.job))433 case .progress(let update, let step, let count):434 let overall = update.fraction.map { (Double(step) + min(max($0, 0), 1)) / Double(max(count, 1)) }435 record.job.progress = JobProgress(fraction: overall, message: update.message, stepIndex: step, stepCount: count)436 let elapsed = record.job.startedAt.map { Date().timeIntervalSince($0) } ?? 0437 record.job.estimatedSecondsRemaining = ETAEstimator.remaining(fraction: overall, elapsed: elapsed)438 records[id] = record439 emit(.jobProgress(id: id, progress: record.job.progress, estimatedSecondsRemaining: record.job.estimatedSecondsRemaining))440 case .log(let level, let message):441 let line = JobLogLine(level: level, message: message)442 append(line, to: &record.job)443 records[id] = record444 emit(.jobLog(id: id, line: line))445 case .warning(let warning):446 if !record.job.warnings.contains(warning) { record.job.warnings.append(warning) }447 let line = JobLogLine(level: .warning, message: warning.message)448 append(line, to: &record.job)449 records[id] = record450 emit(.jobWarning(id: id, warning: warning))451 }452 }453454 private func append(_ line: JobLogLine, to job: inout ConversionJob) {455 job.log.append(line)456 let overflow = job.log.count - configuration.maxLogLinesPerJob457 if overflow > 0 { job.log.removeFirst(overflow) }458 }459460 private func complete(_ id: UUID, attempt: Int, report: RunReport) {461 guard var record = records[id], record.job.attempt == attempt else {462 if case .conflict(_, let held?) = report.outcome { workDirectories.remove(held.workDirectory) }463 return464 }465 record.task = nil466 record.job.checks = report.checks467 record.job.outputs += report.outputs468 record.job.facts = report.facts469 record.job.bytesAfter = report.bytesAfter470 for warning in report.warnings where !record.job.warnings.contains(warning) { record.job.warnings.append(warning) }471 record.job.estimatedSecondsRemaining = nil472 var state = JobState.finished473 var message = "Finished."474 switch report.outcome {475 case .finished:476 record.job.progress.fraction = 1477 case .skipped(let reason):478 state = .skipped479 message = "Skipped: \(reason)"480 case .cancelled:481 state = .cancelled482 message = "Cancelled."483 case .failed(let error):484 state = .failed485 record.job.failure = error486 message = "Failed: \(error.errorDescription ?? "unknown error")"487 case .conflict(let conflict, let held):488 record.held = held489 record.job.conflict = conflict490 record.job.state = .awaitingDecision491 append(JobLogLine(level: .info, message: "Waiting for a decision: \(conflict.existing.lastPathComponent) already exists."), to: &record.job)492 let waiters = record.waiters493 record.waiters = []494 records[id] = record495 emit(.jobChanged(record.job))496 waiters.forEach { $0.resume(returning: record.job) }497 pump()498 return499 }500 records[id] = record501 settle(id, state: state, log: message, level: state == .failed ? .error : .info)502 }503504 /// Puts a job in a terminal state, reschedules, records it in the history and then wakes the waiters (so a505 /// caller returning from `waitForJob` finds the history row written).506 private func settle(_ id: UUID, state: JobState, log message: String, level: LogLevel = .info) {507 guard var record = records[id] else { return }508 record.job.state = state509 record.job.finishedAt = Date()510 append(JobLogLine(level: level, message: message), to: &record.job)511 let waiters = record.waiters512 record.waiters = []513 records[id] = record514 let job = record.job515 emit(.jobChanged(job))516 if job.request.recordInHistory, let history {517 pendingHistory[id, default: 0] += 1518 let entry = Self.historyEntry(for: job)519 Task {520 _ = try? await history.record(entry)521 waiters.forEach { $0.resume(returning: job) }522 self.historyWritten(id)523 }524 } else {525 waiters.forEach { $0.resume(returning: job) }526 }527 pump()528 }529530 private func historyWritten(_ id: UUID) {531 if let count = pendingHistory[id] { pendingHistory[id] = count > 1 ? count - 1 : nil }532 // Waiters that arrived while the row was being written.533 if pendingHistory[id] == nil, let record = records[id], record.job.state.isSettled, record.task == nil, !record.waiters.isEmpty {534 let waiters = record.waiters535 records[id]?.waiters = []536 waiters.forEach { $0.resume(returning: record.job) }537 }538 publishStatus()539 }540541 private func transition(_ id: UUID, to state: JobState, log message: String) {542 guard var record = records[id] else { return }543 record.job.state = state544 append(JobLogLine(level: .info, message: message), to: &record.job)545 records[id] = record546 emit(.jobChanged(record.job))547 }548549 static func historyEntry(for job: ConversionJob) -> HistoryEntry {550 var entry = HistoryEntry(jobID: job.id, sourcePath: job.request.sources.first?.path ?? "", destinationFormat: job.request.destinationFormat,551 status: job.state.rawValue, createdAt: job.createdAt, finishedAt: job.finishedAt ?? Date())552 entry.additionalSourcePaths = job.request.sources.dropFirst().map(\.path)553 entry.sourceFormat = job.sourceFormat554 entry.destinationPaths = job.outputs.filter { $0.outcome != .skipped }.map(\.url.path)555 entry.operation = job.request.operation556 entry.engine = job.plan?.engines.joined(separator: "+") ?? job.request.engineID ?? ""557 entry.options = job.request.options.redactingSecrets()558 entry.bytesBefore = job.bytesBefore559 entry.bytesAfter = job.bytesAfter560 entry.duration = job.duration561 entry.warnings = job.warnings562 entry.checks = job.checks563 entry.error = job.error564 entry.startedAt = job.startedAt565 entry.batchID = job.request.batchID566 entry.workflowRunID = job.request.workflowRunID567 return entry568 }569570 // MARK: Events571572 private func emit(_ event: QueueEvent) {573 for continuation in subscribers.values { continuation.yield(event) }574 }575576 private func unsubscribe(_ token: UUID) {577 subscribers[token] = nil578 }579}580581/// Keeps NotificationCenter observers alive for the lifetime of their owner.582final class ObserverBag: @unchecked Sendable {583 private let lock = NSLock()584 private var tokens: [NSObjectProtocol] = []585 func add(_ token: NSObjectProtocol) { lock.lock(); tokens.append(token); lock.unlock() }586 deinit { tokens.forEach(NotificationCenter.default.removeObserver) }587}588