SPB Git

Repository Public License

ConversionQueue.swift 26.6 KB 588 lines swift

1// ORIX Convert — Packages/ORIXConvertCore/Sources/ConvertCore/Jobs/ConversionQueue.swift2// Description: The conversion queue: an actor that owns every job, schedules them by priority within the resource3//              limits (concurrency, heavy class, thermal and power state), and publishes events and snapshots.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation67/// The job queue.8///9/// **Pause semantics.** `pause()` stops new jobs from starting; jobs already running continue to their end.10/// `pauseJob` holds back a job that has not started. A running job cannot be paused: engines offer no cooperative11/// suspension point, so the queue does not pretend to — cancel it and retry later instead.12///13/// **Cancellation.** `cancel` cancels the job's task. Engines check `context.checkCancellation()` in their loops14/// and `ProcessRunner` terminates helper processes; the job becomes `cancelled` when the engine returns.15///16/// **Errors.** A failed job never affects the other jobs of its batch.17public actor ConversionQueue {18    public nonisolated let registry: FormatRegistry19    private var configuration: QueueConfiguration20    private let history: ConversionHistoryStore?21    private let workDirectories: WorkDirectoryManager22    private let monitor: any ResourceMonitoring2324    private struct Record {25        var job: ConversionJob26        var task: Task<Void, Never>?27        var held: HeldOutput?28        var policy: ConflictPolicy29        var allowReplacingSource: Bool30        var waiters: [CheckedContinuation<ConversionJob, Never>] = []31    }3233    private var records: [UUID: Record] = [:]34    private var order: [UUID] = []35    private var paused = false36    private var subscribers: [UUID: AsyncStream<QueueEvent>.Continuation] = [:]37    private var idleWaiters: [CheckedContinuation<Void, Never>] = []38    private var lastStatus: QueueStatus?39    private var wasIdle = true40    private var pendingHistory: [UUID: Int] = [:]41    private let observers = ObserverBag()4243    /// - Parameters:44    ///   - history: when given, every job that ends (finished, failed, cancelled, skipped) is recorded.45    ///   - workDirectories: where engines write. Directories left by dead processes are swept now.46    ///   - monitor: thermal / power state source (the system by default).47    public init(registry: FormatRegistry, configuration: QueueConfiguration = QueueConfiguration(), history: ConversionHistoryStore? = nil,48                workDirectories: WorkDirectoryManager = WorkDirectoryManager(), monitor: any ResourceMonitoring = SystemResourceMonitor()) {49        self.registry = registry50        self.configuration = configuration51        self.history = history52        self.workDirectories = workDirectories53        self.monitor = monitor54        workDirectories.sweepStale()55        let center = NotificationCenter.default56        for name in [ProcessInfo.thermalStateDidChangeNotification, Notification.Name.NSProcessInfoPowerStateDidChange] {57            observers.add(center.addObserver(forName: name, object: nil, queue: nil) { [weak self] _ in58                Task { await self?.resourceStateChanged() }59            })60        }61    }6263    // MARK: Enqueueing6465    /// Adds a job and returns its identifier. The job starts as soon as a slot is free.66    @discardableResult67    public func enqueue(_ request: ConversionJobRequest) -> UUID {68        let id = UUID()69        let job = ConversionJob(70            id: id, request: request, state: .queued, priority: request.priority, sourceFormat: request.sourceFormat, detection: nil,71            plan: nil, progress: JobProgress(), estimatedSecondsRemaining: nil,72            log: [JobLogLine(level: .info, message: "Queued.")], warnings: [], checks: [], outputs: [], facts: [:], failure: nil,73            conflict: nil, createdAt: Date(), startedAt: nil, finishedAt: nil, attempt: 1, isHeavy: isHeavy(request), bytesBefore: nil,74            bytesAfter: nil, cancelRequested: false)75        records[id] = Record(job: job, policy: request.destination.conflictPolicy, allowReplacingSource: request.destination.allowReplacingSource)76        order.append(id)77        emit(.jobAdded(job))78        pump()79        return id80    }8182    /// Adds several jobs as one batch: they share a `batchID` and get counters 1…n (unless already set).83    @discardableResult84    public func enqueue(batch requests: [ConversionJobRequest]) -> [UUID] {85        let batchID = UUID()86        return requests.enumerated().map { index, request in87            var request = request88            request.batchID = request.batchID ?? batchID89            if request.counter <= 1 { request.counter = index + 1 }90            return enqueue(request)91        }92    }9394    // MARK: Control9596    /// Stops starting new jobs. Running jobs continue.97    public func pause() {98        guard !paused else { return }99        paused = true100        publishStatus()101    }102103    public func resume() {104        guard paused else { return }105        paused = false106        pump()107    }108109    public var isPaused: Bool { paused }110111    /// Holds back a job that has not started. Returns `false` for a job that is running (see the type's notes),112    /// settled or unknown.113    @discardableResult114    public func pauseJob(_ id: UUID) -> Bool {115        guard records[id]?.job.state == .queued else { return false }116        transition(id, to: .paused, log: "Paused before start.")117        pump()118        return true119    }120121    @discardableResult122    public func resumeJob(_ id: UUID) -> Bool {123        guard records[id]?.job.state == .paused else { return false }124        transition(id, to: .queued, log: "Resumed.")125        pump()126        return true127    }128129    /// Cancels a job. A waiting job is cancelled at once; a running job is cancelled cooperatively.130    @discardableResult131    public func cancel(_ id: UUID) -> Bool {132        guard var record = records[id] else { return false }133        switch record.job.state {134        case .queued, .paused, .awaitingDecision:135            if let held = record.held { workDirectories.remove(held.workDirectory) }136            record.held = nil137            record.job.conflict = nil138            records[id] = record139            settle(id, state: .cancelled, log: "Cancelled.")140            return true141        case .preparing, .running, .validating:142            record.job.cancelRequested = true143            record.job.log.append(JobLogLine(level: .info, message: "Cancellation requested."))144            records[id] = record145            record.task?.cancel()146            emit(.jobChanged(record.job))147            return true148        case .finished, .failed, .cancelled, .skipped:149            return false150        }151    }152153    /// Cancels every job that is not settled.154    public func cancelAll() {155        for id in order { cancel(id) }156    }157158    /// Runs a failed, cancelled or skipped job again (same identifier, `attempt` incremented).159    @discardableResult160    public func retry(_ id: UUID) -> Bool {161        guard var record = records[id], [.failed, .cancelled, .skipped].contains(record.job.state), record.task == nil else { return false }162        record.job.attempt += 1163        record.job.state = .queued164        record.job.failure = nil165        record.job.conflict = nil166        record.job.checks = []167        record.job.warnings = []168        record.job.outputs = []169        record.job.facts = [:]170        record.job.progress = JobProgress()171        record.job.estimatedSecondsRemaining = nil172        record.job.startedAt = nil173        record.job.finishedAt = nil174        record.job.cancelRequested = false175        record.policy = record.job.request.destination.conflictPolicy176        record.allowReplacingSource = record.job.request.destination.allowReplacingSource177        record.job.log.append(JobLogLine(level: .info, message: "Retry (attempt \(record.job.attempt))."))178        records[id] = record179        emit(.jobChanged(record.job))180        pump()181        return true182    }183184    /// Changes the priority of a job that has not started.185    @discardableResult186    public func setPriority(_ priority: JobPriority, for id: UUID) -> Bool {187        guard var record = records[id], record.job.state == .queued || record.job.state == .paused else { return false }188        record.job.priority = priority189        records[id] = record190        emit(.jobChanged(record.job))191        return true192    }193194    /// Moves a job to `index` in the display order (which is the start order within a priority).195    @discardableResult196    public func move(_ id: UUID, to index: Int) -> Bool {197        guard let current = order.firstIndex(of: id) else { return false }198        order.remove(at: current)199        order.insert(id, at: min(max(0, index), order.count))200        if let job = records[id]?.job { emit(.jobChanged(job)) }201        return true202    }203204    /// Removes a job from the list. Running jobs must be cancelled first.205    @discardableResult206    public func remove(_ id: UUID) -> Bool {207        guard let record = records[id], !record.job.state.isActive, record.task == nil else { return false }208        if let held = record.held { workDirectories.remove(held.workDirectory) }209        for waiter in record.waiters { waiter.resume(returning: record.job) }210        records[id] = nil211        order.removeAll { $0 == id }212        emit(.jobRemoved(id))213        pump()214        return true215    }216217    /// Removes every job in a terminal state. Returns how many were removed.218    @discardableResult219    public func clearSettled() -> Int {220        let ids = order.filter { records[$0]?.job.state.isTerminal == true && records[$0]?.task == nil }221        ids.forEach { remove($0) }222        return ids.count223    }224225    /// Decides a job in `awaitingDecision`.226    ///227    /// - `rename` / `replace`: writes the output (or starts the conversion when the conflict was found beforehand)228    ///   under that policy. Replacing the job's own source additionally needs `allowReplacingSource`.229    /// - `skip`: keeps the existing file; the job ends `skipped`.230    /// - `askUser` is not a decision and returns `false`.231    @discardableResult232    public func resolveConflict(_ id: UUID, policy: ConflictPolicy, allowReplacingSource: Bool = false) -> Bool {233        guard var record = records[id], record.job.state == .awaitingDecision, policy != .askUser else { return false }234        record.policy = policy235        record.allowReplacingSource = allowReplacingSource || record.allowReplacingSource236        let conflict = record.job.conflict237        record.job.conflict = nil238        guard let held = record.held else {239            // Conflict found before converting.240            records[id] = record241            if policy == .skip {242                settle(id, state: .skipped, log: "Skipped: the existing file was kept.")243            } else {244                transition(id, to: .queued, log: "Decision: \(policy.rawValue).")245                pump()246            }247            return true248        }249        record.held = nil250        records[id] = record251        if policy == .skip {252            workDirectories.remove(held.workDirectory)253            settle(id, state: record.job.outputs.contains { $0.outcome != .skipped } ? .finished : .skipped, log: "Skipped: the existing file was kept.")254            return true255        }256        do {257            let committed = try SafeWriter.commit(held.items, names: held.names, to: held.directory, policy: policy, sources: record.job.request.sources,258                                                  allowReplacingSource: record.allowReplacingSource)259            workDirectories.remove(held.workDirectory)260            records[id]?.job.outputs += committed261            let written = (records[id]?.job.outputs ?? []).filter { $0.outcome != .skipped }262            records[id]?.job.bytesAfter = written.compactMap { FileSizer.size(of: $0.url) }.reduce(Int64(0), +)263            settle(id, state: .finished, log: "Decision: \(policy.rawValue). Wrote \(committed.map(\.url.lastPathComponent).joined(separator: ", ")).")264        } catch ConversionError.destinationExists {265            // Still not allowed (e.g. replacing the source without permission): keep waiting.266            records[id]?.held = held267            records[id]?.job.conflict = conflict268            return false269        } catch {270            workDirectories.remove(held.workDirectory)271            records[id]?.job.failure = (error as? ConversionError) ?? .writeFailed(error.localizedDescription)272            settle(id, state: .failed, log: "Writing failed: \(error.localizedDescription)")273        }274        return true275    }276277    /// Replaces the configuration (limits apply to the next scheduling decision).278    public func updateConfiguration(_ configuration: QueueConfiguration) {279        self.configuration = configuration280        pump()281    }282283    public var currentConfiguration: QueueConfiguration { configuration }284285    /// Re-reads the thermal and power state and reschedules. Called automatically on thermal and Low Power Mode286    /// notifications; the battery state is re-read at every scheduling decision.287    public func resourceStateChanged() {288        pump()289    }290291    // MARK: Observation292293    public func snapshot() -> QueueSnapshot {294        QueueSnapshot(jobs: order.compactMap { records[$0]?.job }, status: status())295    }296297    public func job(_ id: UUID) -> ConversionJob? { records[id]?.job }298299    /// A stream of everything that happens in the queue, from now on. Each call returns an independent stream;300    /// cancel the consuming task (or drop the stream) to unsubscribe. A slow consumer loses the oldest events301    /// beyond 4096 buffered ones — use `snapshot()` to resynchronise.302    public func events() -> AsyncStream<QueueEvent> {303        let token = UUID()304        let (stream, continuation) = AsyncStream<QueueEvent>.makeStream(bufferingPolicy: .bufferingNewest(4096))305        subscribers[token] = continuation306        continuation.onTermination = { [weak self] _ in307            Task { await self?.unsubscribe(token) }308        }309        return stream310    }311312    /// Waits until the job is settled (finished, failed, cancelled, skipped, or awaiting a decision) and returns its313    /// snapshot; `nil` for an unknown job.314    public func waitForJob(_ id: UUID) async -> ConversionJob? {315        guard let record = records[id] else { return nil }316        if record.job.state.isSettled, record.task == nil, pendingHistory[id] == nil { return record.job }317        return await withCheckedContinuation { continuation in318            records[id]?.waiters.append(continuation)319        }320    }321322    /// Waits until nothing is running and nothing can start (queue empty of startable jobs, or paused).323    public func waitUntilIdle() async {324        if isIdle { return }325        await withCheckedContinuation { idleWaiters.append($0) }326    }327328    // MARK: Scheduling329330    private var isIdle: Bool {331        let jobs = records.values332        if !pendingHistory.isEmpty || jobs.contains(where: { $0.job.state.isActive || $0.task != nil }) { return false }333        return paused || !jobs.contains { $0.job.state == .queued }334    }335336    private func status() -> QueueStatus {337        let state = monitor.currentState()338        let jobs = records.values.map(\.job)339        return QueueStatus(isPaused: paused, limits: configuration.limits(for: state), resources: state,340                           queued: jobs.filter { $0.state == .queued || $0.state == .paused }.count,341                           active: jobs.filter(\.state.isActive).count, settled: jobs.filter(\.state.isSettled).count)342    }343344    private func publishStatus() {345        let current = status()346        if current != lastStatus {347            lastStatus = current348            emit(.queueChanged(current))349        }350        let idle = isIdle351        if idle {352            if !wasIdle { emit(.idle) }353            let waiters = idleWaiters354            idleWaiters = []355            waiters.forEach { $0.resume() }356        }357        wasIdle = idle358    }359360    /// Starts as many queued jobs as the limits allow: highest priority first, then display order. A heavy job that361    /// cannot start because the heavy slots are taken does not block lighter jobs behind it.362    private func pump() {363        defer { publishStatus() }364        guard !paused else { return }365        let limits = configuration.limits(for: monitor.currentState())366        var total = records.values.filter { $0.job.state.isActive }.count367        var heavy = records.values.filter { $0.job.state.isActive && $0.job.isHeavy }.count368        let candidates = order.enumerated()369            .compactMap { index, id in records[id].map { (index, $0.job) } }370            .filter { $0.1.state == .queued && records[$0.1.id]?.task == nil }371            .sorted { ($1.1.priority.rawValue, $0.0) < ($0.1.priority.rawValue, $1.0) }372        for (_, job) in candidates {373            guard total < limits.total else { break }374            if job.isHeavy {375                guard heavy < limits.heavy else { continue }376                heavy += 1377            }378            total += 1379            launch(job.id)380        }381    }382383    private func isHeavy(_ request: ConversionJobRequest) -> Bool {384        if let engine = request.engineID, configuration.heavyEngineIDs.contains(engine) { return true }385        let guessed = request.sourceFormat ?? request.sources.first.flatMap { FormatCatalog.formats(forExtension: $0.pathExtension).first?.id }386        if [guessed, request.destinationFormat].contains(where: { $0.flatMap(FormatCatalog.format)?.category == .video }) { return true }387        if let guessed, let plan = try? ConversionPlanner(registry: registry).plan(from: guessed, to: request.destinationFormat, operation: request.operation,388                                                                                   engine: request.engineID, allowChaining: request.allowChaining),389           !configuration.heavyEngineIDs.isDisjoint(with: plan.engines) { return true }390        let size = request.sources.reduce(Int64(0)) { total, url in391            total + (((try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? NSNumber)?.int64Value ?? 0)392        }393        return size >= configuration.heavyFileSize394    }395396    private func launch(_ id: UUID) {397        guard var record = records[id] else { return }398        record.job.state = .preparing399        record.job.startedAt = Date()400        record.job.log.append(JobLogLine(level: .info, message: "Started."))401        let attempt = record.job.attempt402        let (stream, continuation) = AsyncStream<JobSignal>.makeStream(bufferingPolicy: .bufferingNewest(8192))403        let runner = JobRunner(id: id, request: record.job.request, policy: record.policy, allowReplacingSource: record.allowReplacingSource,404                               registry: registry, workDirectories: workDirectories, signal: { continuation.yield($0) })405        record.task = Task {406            let forwarder = Task {407                for await signal in stream { self.apply(signal, to: id, attempt: attempt) }408            }409            let report = await runner.run()410            continuation.finish()411            await forwarder.value412            self.complete(id, attempt: attempt, report: report)413        }414        records[id] = record415        emit(.jobChanged(record.job))416    }417418    private func apply(_ signal: JobSignal, to id: UUID, attempt: Int) {419        guard var record = records[id], record.job.attempt == attempt, record.task != nil else { return }420        switch signal {421        case .state(let state):422            record.job.state = state423            records[id] = record424            emit(.jobChanged(record.job))425        case .prepared(let sourceFormat, let detection, let plan, let bytesBefore):426            record.job.sourceFormat = sourceFormat427            record.job.detection = detection428            record.job.plan = plan429            record.job.bytesBefore = bytesBefore430            record.job.progress = JobProgress(stepIndex: 0, stepCount: plan.steps.count)431            records[id] = record432            emit(.jobChanged(record.job))433        case .progress(let update, let step, let count):434            let overall = update.fraction.map { (Double(step) + min(max($0, 0), 1)) / Double(max(count, 1)) }435            record.job.progress = JobProgress(fraction: overall, message: update.message, stepIndex: step, stepCount: count)436            let elapsed = record.job.startedAt.map { Date().timeIntervalSince($0) } ?? 0437            record.job.estimatedSecondsRemaining = ETAEstimator.remaining(fraction: overall, elapsed: elapsed)438            records[id] = record439            emit(.jobProgress(id: id, progress: record.job.progress, estimatedSecondsRemaining: record.job.estimatedSecondsRemaining))440        case .log(let level, let message):441            let line = JobLogLine(level: level, message: message)442            append(line, to: &record.job)443            records[id] = record444            emit(.jobLog(id: id, line: line))445        case .warning(let warning):446            if !record.job.warnings.contains(warning) { record.job.warnings.append(warning) }447            let line = JobLogLine(level: .warning, message: warning.message)448            append(line, to: &record.job)449            records[id] = record450            emit(.jobWarning(id: id, warning: warning))451        }452    }453454    private func append(_ line: JobLogLine, to job: inout ConversionJob) {455        job.log.append(line)456        let overflow = job.log.count - configuration.maxLogLinesPerJob457        if overflow > 0 { job.log.removeFirst(overflow) }458    }459460    private func complete(_ id: UUID, attempt: Int, report: RunReport) {461        guard var record = records[id], record.job.attempt == attempt else {462            if case .conflict(_, let held?) = report.outcome { workDirectories.remove(held.workDirectory) }463            return464        }465        record.task = nil466        record.job.checks = report.checks467        record.job.outputs += report.outputs468        record.job.facts = report.facts469        record.job.bytesAfter = report.bytesAfter470        for warning in report.warnings where !record.job.warnings.contains(warning) { record.job.warnings.append(warning) }471        record.job.estimatedSecondsRemaining = nil472        var state = JobState.finished473        var message = "Finished."474        switch report.outcome {475        case .finished:476            record.job.progress.fraction = 1477        case .skipped(let reason):478            state = .skipped479            message = "Skipped: \(reason)"480        case .cancelled:481            state = .cancelled482            message = "Cancelled."483        case .failed(let error):484            state = .failed485            record.job.failure = error486            message = "Failed: \(error.errorDescription ?? "unknown error")"487        case .conflict(let conflict, let held):488            record.held = held489            record.job.conflict = conflict490            record.job.state = .awaitingDecision491            append(JobLogLine(level: .info, message: "Waiting for a decision: \(conflict.existing.lastPathComponent) already exists."), to: &record.job)492            let waiters = record.waiters493            record.waiters = []494            records[id] = record495            emit(.jobChanged(record.job))496            waiters.forEach { $0.resume(returning: record.job) }497            pump()498            return499        }500        records[id] = record501        settle(id, state: state, log: message, level: state == .failed ? .error : .info)502    }503504    /// Puts a job in a terminal state, reschedules, records it in the history and then wakes the waiters (so a505    /// caller returning from `waitForJob` finds the history row written).506    private func settle(_ id: UUID, state: JobState, log message: String, level: LogLevel = .info) {507        guard var record = records[id] else { return }508        record.job.state = state509        record.job.finishedAt = Date()510        append(JobLogLine(level: level, message: message), to: &record.job)511        let waiters = record.waiters512        record.waiters = []513        records[id] = record514        let job = record.job515        emit(.jobChanged(job))516        if job.request.recordInHistory, let history {517            pendingHistory[id, default: 0] += 1518            let entry = Self.historyEntry(for: job)519            Task {520                _ = try? await history.record(entry)521                waiters.forEach { $0.resume(returning: job) }522                self.historyWritten(id)523            }524        } else {525            waiters.forEach { $0.resume(returning: job) }526        }527        pump()528    }529530    private func historyWritten(_ id: UUID) {531        if let count = pendingHistory[id] { pendingHistory[id] = count > 1 ? count - 1 : nil }532        // Waiters that arrived while the row was being written.533        if pendingHistory[id] == nil, let record = records[id], record.job.state.isSettled, record.task == nil, !record.waiters.isEmpty {534            let waiters = record.waiters535            records[id]?.waiters = []536            waiters.forEach { $0.resume(returning: record.job) }537        }538        publishStatus()539    }540541    private func transition(_ id: UUID, to state: JobState, log message: String) {542        guard var record = records[id] else { return }543        record.job.state = state544        append(JobLogLine(level: .info, message: message), to: &record.job)545        records[id] = record546        emit(.jobChanged(record.job))547    }548549    static func historyEntry(for job: ConversionJob) -> HistoryEntry {550        var entry = HistoryEntry(jobID: job.id, sourcePath: job.request.sources.first?.path ?? "", destinationFormat: job.request.destinationFormat,551                                 status: job.state.rawValue, createdAt: job.createdAt, finishedAt: job.finishedAt ?? Date())552        entry.additionalSourcePaths = job.request.sources.dropFirst().map(\.path)553        entry.sourceFormat = job.sourceFormat554        entry.destinationPaths = job.outputs.filter { $0.outcome != .skipped }.map(\.url.path)555        entry.operation = job.request.operation556        entry.engine = job.plan?.engines.joined(separator: "+") ?? job.request.engineID ?? ""557        entry.options = job.request.options.redactingSecrets()558        entry.bytesBefore = job.bytesBefore559        entry.bytesAfter = job.bytesAfter560        entry.duration = job.duration561        entry.warnings = job.warnings562        entry.checks = job.checks563        entry.error = job.error564        entry.startedAt = job.startedAt565        entry.batchID = job.request.batchID566        entry.workflowRunID = job.request.workflowRunID567        return entry568    }569570    // MARK: Events571572    private func emit(_ event: QueueEvent) {573        for continuation in subscribers.values { continuation.yield(event) }574    }575576    private func unsubscribe(_ token: UUID) {577        subscribers[token] = nil578    }579}580581/// Keeps NotificationCenter observers alive for the lifetime of their owner.582final class ObserverBag: @unchecked Sendable {583    private let lock = NSLock()584    private var tokens: [NSObjectProtocol] = []585    func add(_ token: NSObjectProtocol) { lock.lock(); tokens.append(token); lock.unlock() }586    deinit { tokens.forEach(NotificationCenter.default.removeObserver) }587}588
Files main
Menu

Theme