SPB Git

Repository Public License

ConvertCommand.swift 26.9 KB 479 lines swift

1// ORIX Convert — Packages/ConvertKit/Sources/ConvertCLI/ConvertCommand.swift2// Description: `orix-convert convert`: turns files into queue jobs, follows them through the real queue (detector,3//              planner, engine, validation, safe writer), prints measured progress, warnings, checks and a summary.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import ConvertCore7import ConvertKit89/// Result of one job, as printed and as exported in JSON.10public struct JobReport: Codable, Sendable {11    public struct Output: Codable, Sendable { public let path: String; public let outcome: String }12    public var sources: [String]13    public var sourceFormat: String?14    public var destinationFormat: String15    public var operation: String?16    public var state: String17    public var engines: [String]18    public var chained: Bool19    public var outputs: [Output]20    public var warnings: [ConversionWarning]21    public var checks: [ValidationCheck]22    public var facts: [String: String]23    public var error: String?24    public var errorKind: String?25    public var seconds: Double?26    public var bytesBefore: Int64?27    public var bytesAfter: Int64?28}2930public struct BatchSummary: Codable, Sendable {31    public var total = 0, finished = 0, failed = 0, skipped = 0, cancelled = 032    public var bytesBefore: Int64 = 0, bytesAfter: Int64 = 033    public var seconds = 0.034    public var interrupted = false35    public var exitCode: Int32 = 036}3738public struct BatchReport: Codable, Sendable {39    public var summary: BatchSummary40    public var jobs: [JobReport]41    /// Work folders of this process still present after the run (must be empty).42    public var leftoverTemporaryItems: [String]43}4445/// Runs a batch of job requests on a service's queue and reports as jobs settle.46struct BatchRunner: Sendable {47    struct Display: Sendable {48        var quiet = false49        var verbose = false50        var json = false51    }5253    let service: ConversionService54    let io: CommandIO55    let display: Display56    /// `--on-conflict fail`: jobs run with `askUser`; a job that stops on an existing destination is cancelled and57    /// reported as failed.58    let failOnConflict: Bool5960    func run(_ requests: [ConversionJobRequest], afterEnqueue: (@Sendable ([UUID]) async -> Void)? = nil) async -> BatchReport {61        let started = Date()62        let screen = Screen(io: io, showsProgress: !display.quiet && !display.json)63        let queue = service.queue64        let total = requests.count65        let tracker = ProgressTracker(total: total)6667        let events = await queue.events()68        let observer = Task {69            for await event in events {70                switch event {71                case .jobChanged(let job):72                    tracker.update(job.id, name: job.primarySource?.lastPathComponent ?? "", state: job.state, fraction: job.progress.fraction, eta: job.estimatedSecondsRemaining)73                case .jobProgress(let id, let progress, let eta):74                    tracker.progress(id, fraction: progress.fraction, eta: eta, step: progress.stepIndex, steps: progress.stepCount)75                default:76                    continue77                }78                screen.progress(tracker.line())79            }80        }81        let interrupt = InterruptCenter.shared.register {82            Task { await queue.cancelAll() }83        }84        defer { InterruptCenter.shared.unregister(interrupt) }8586        let ids = await queue.enqueue(batch: requests)87        if let afterEnqueue { await afterEnqueue(ids) }8889        var reports = [JobReport?](repeating: nil, count: ids.count)90        await withTaskGroup(of: (Int, JobReport).self) { group in91            for (index, id) in ids.enumerated() {92                group.addTask { (index, await self.settle(id, request: requests[index])) }93            }94            for await (index, report) in group {95                reports[index] = report96                tracker.settled()97                if !display.json { print(report, index: index + 1, total: total, screen: screen) }98            }99        }100        observer.cancel()101        screen.clear()102103        var summary = BatchSummary()104        let jobs = reports.compactMap { $0 }105        summary.total = jobs.count106        for job in jobs {107            switch job.state {108            case JobState.finished.rawValue: summary.finished += 1109            case JobState.skipped.rawValue: summary.skipped += 1110            case JobState.cancelled.rawValue: summary.cancelled += 1111            default: summary.failed += 1112            }113            summary.bytesBefore += job.bytesBefore ?? 0114            summary.bytesAfter += job.bytesAfter ?? 0115        }116        summary.seconds = Date().timeIntervalSince(started)117        summary.interrupted = InterruptCenter.shared.isInterrupted118        summary.exitCode = Self.exitCode(jobs: jobs, interrupted: summary.interrupted).rawValue119        let prefix = "\(ProcessInfo.processInfo.processIdentifier)-"120        return BatchReport(summary: summary, jobs: jobs, leftoverTemporaryItems: service.workRootLeftovers().filter { $0.hasPrefix(prefix) })121    }122123    /// Waits for a job to settle and turns the outcome into a report.124    private func settle(_ id: UUID, request: ConversionJobRequest) async -> JobReport {125        let queue = service.queue126        var conflictMessage: String?127        var job = await queue.waitForJob(id)128        if let waiting = job, waiting.state == .awaitingDecision {129            let existing = waiting.conflict?.existing.path ?? "the destination"130            conflictMessage = "\(existing) already exists. Nothing was written (use --on-conflict rename, skip or replace)."131            await queue.cancel(id)132            job = await queue.waitForJob(id)133        }134        guard let job else {135            return JobReport(sources: request.sources.map(\.path), sourceFormat: nil, destinationFormat: request.destinationFormat.rawValue,136                             operation: request.operation, state: JobState.failed.rawValue, engines: [], chained: false, outputs: [], warnings: [],137                             checks: [], facts: [:], error: "The job disappeared from the queue.", errorKind: "internalError", seconds: nil,138                             bytesBefore: nil, bytesAfter: nil)139        }140        var state = job.state.rawValue141        var error = job.error142        var kind = job.failure?.kind143        if let conflictMessage {144            state = JobState.failed.rawValue; error = conflictMessage; kind = "destinationExists"145        }146        if job.state == .skipped, error == nil {147            error = job.log.last(where: { $0.message.contains("exist") || $0.message.contains("Skipped") })?.message148        }149        return JobReport(sources: job.request.sources.map(\.path), sourceFormat: job.sourceFormat?.rawValue,150                         destinationFormat: job.request.destinationFormat.rawValue, operation: job.request.operation, state: state,151                         engines: job.plan?.engines ?? [], chained: job.plan?.isChained ?? false,152                         outputs: job.outputs.map { JobReport.Output(path: $0.url.path, outcome: $0.outcome.rawValue) },153                         warnings: job.warnings, checks: job.checks, facts: job.facts, error: error, errorKind: kind, seconds: job.duration,154                         bytesBefore: job.bytesBefore, bytesAfter: job.bytesAfter)155    }156157    private func print(_ report: JobReport, index: Int, total: Int, screen: Screen) {158        let name = report.sources.first.map { ($0 as NSString).lastPathComponent } ?? "?"159        let more = report.sources.count > 1 ? " (+\(report.sources.count - 1) more)" : ""160        switch report.state {161        case JobState.finished.rawValue:162            guard !display.quiet else { return }163            var details: [String] = []164            if let source = report.sourceFormat { details.append("\(source) → \(report.destinationFormat)" + (report.operation.map { " [\($0)]" } ?? "")) }165            if !report.engines.isEmpty { details.append((report.chained ? "chained: " : "") + report.engines.joined(separator: " + ")) }166            if let before = report.bytesBefore, let after = report.bytesAfter { details.append("\(Shared.bytes(before)) → \(Shared.bytes(after))") }167            if let seconds = report.seconds { details.append(Shared.seconds(seconds)) }168            let written = report.outputs.filter { $0.outcome != "skipped" }169            let target = written.count == 1 ? written[0].path : "\(written.count) items in \((written.first?.path as NSString?)?.deletingLastPathComponent ?? "")"170            screen.line("ok       \(name)\(more) → \(target)  (\(details.joined(separator: ", ")))")171            for output in report.outputs where output.outcome != "written" {172                screen.line("           \(output.outcome): \(output.path)")173            }174        case JobState.skipped.rawValue:175            guard !display.quiet else { return }176            screen.line("skipped  \(name)\(more): \(report.error ?? "the destination already exists.")")177        case JobState.cancelled.rawValue:178            screen.error("cancelled \(name)\(more)")179        default:180            screen.error("FAILED   \(name)\(more): \(report.error ?? "unknown error")")181        }182        guard !display.quiet else { return }183        for warning in report.warnings {184            screen.line("           warning [\(warning.severity.rawValue)] \(warning.code): \(warning.message)")185        }186        if !report.checks.isEmpty {187            let failed = report.checks.filter { !$0.passed }188            if display.verbose || !failed.isEmpty {189                for check in report.checks where display.verbose || !check.passed {190                    screen.line("           check \(check.passed ? "passed" : "FAILED") \(check.name)\(check.detail.isEmpty ? "" : ": \(check.detail)")")191                }192            } else {193                let names = Array(Set(report.checks.map(\.name))).sorted()194                screen.line("           validated: \(report.checks.count) check(s) passed (\(names.prefix(8).joined(separator: ", "))\(names.count > 8 ? ", …" : ""))")195            }196        }197        if display.verbose {198            for (key, value) in report.facts.sorted(by: { $0.key < $1.key }) { screen.line("           fact \(key) = \(value)") }199        }200    }201202    static func exitCode(jobs: [JobReport], interrupted: Bool) -> ExitCode {203        if interrupted { return .interrupted }204        let failures = jobs.filter { $0.state != JobState.finished.rawValue && $0.state != JobState.skipped.rawValue }205        guard !failures.isEmpty else { return .success }206        let kinds = Set(failures.map { $0.errorKind ?? "" })207        if kinds == ["unsupportedPath"] { return .unsupported }208        if kinds == ["destinationExists"] { return .destinationExists }209        if kinds.isSubset(of: ["unreadableInput", "malformedInput"]), failures.count == jobs.count { return .inputError }210        return .conversionFailed211    }212213    static func summaryLine(_ summary: BatchSummary) -> String {214        var parts = ["\(summary.finished) converted", "\(summary.failed) failed", "\(summary.skipped) skipped"]215        if summary.cancelled > 0 { parts.append("\(summary.cancelled) cancelled") }216        var line = "Summary: \(summary.total) job(s) — " + parts.joined(separator: ", ")217        if summary.finished > 0 { line += " — \(Shared.bytes(summary.bytesBefore)) in, \(Shared.bytes(summary.bytesAfter)) out" }218        return line + " — \(Shared.seconds(summary.seconds))" + (summary.interrupted ? " — interrupted" : "")219    }220}221222/// Live state behind the progress line.223final class ProgressTracker: @unchecked Sendable {224    private struct Active { var name: String; var fraction: Double?; var eta: Double?; var step = 0; var steps = 1; var state: JobState }225    private let lock = NSLock()226    private let total: Int227    private var done = 0228    private var active: [UUID: Active] = [:]229    private var order: [UUID] = []230231    init(total: Int) { self.total = total }232233    func update(_ id: UUID, name: String, state: JobState, fraction: Double?, eta: Double?) {234        lock.lock(); defer { lock.unlock() }235        if state.isActive {236            if active[id] == nil { order.append(id) }237            var entry = active[id] ?? Active(name: name, fraction: fraction, eta: eta, state: state)238            entry.state = state239            active[id] = entry240        } else {241            active[id] = nil242            order.removeAll { $0 == id }243        }244    }245246    func progress(_ id: UUID, fraction: Double?, eta: Double?, step: Int, steps: Int) {247        lock.lock(); defer { lock.unlock() }248        guard var entry = active[id] else { return }249        entry.fraction = fraction; entry.eta = eta; entry.step = step; entry.steps = steps250        active[id] = entry251    }252253    func settled() { lock.lock(); done += 1; lock.unlock() }254255    /// `[3/10] 2 running · photo.png 42 % · about 12 s left` — or `working (not measurable)` when the engine cannot256    /// measure its progress. A percentage is shown only when an engine reported one.257    func line() -> String {258        lock.lock(); defer { lock.unlock() }259        var text = "[\(done)/\(total)]"260        guard let first = order.first, let entry = active[first] else { return text + " waiting" }261        if active.count > 1 { text += " \(active.count) running ·" }262        text += " \(entry.name)"263        if entry.steps > 1 { text += " step \(entry.step + 1)/\(entry.steps)" }264        if entry.state == .validating {265            text += " verifying"266        } else if entry.state == .preparing {267            text += " preparing"268        } else if let fraction = entry.fraction {269            text += " \(Int((fraction * 100).rounded())) %"270            if let eta = entry.eta, eta >= 1 { text += " · about \(Int(eta.rounded())) s left" }271        } else {272            text += " working (progress not measurable)"273        }274        return text275    }276}277278enum ConvertCommand: Command {279    static let spec = CommandSpec(280        "convert", summary: "Convert files (real queue, validation, safe writer).",281        usage: "convert <files…> --to <format> [--operation <op>] [--option key=value …] [--output-dir <dir> | --output <file>] [--name-template <t>] [--on-conflict rename|skip|replace|fail] [--jobs <n>] [--recursive] [--combine] [--from <format>] [--engine <id>] [--json] [--quiet] [--verbose] [--no-history]",282        flags: ["json", "quiet", "verbose", "no-history", "recursive", "combine", "no-chain"],283        options: ["to", "operation", "output-dir", "output", "name-template", "on-conflict", "jobs", "from", "engine"],284        repeated: ["option"], minPositionals: 1, unbounded: true,285        details: [286            "  --to <format>           destination format (identifier or extension: jpeg, jpg, pdf, folder…)",287            "  --operation <op>        named operation (pdf.merge, image.vectorize, video.extract-audio…); see 'paths'",288            "  --option key=value      a setting of the path (see 'options'); repeatable. For a key containing 'password',",289            "                          the value @env:NAME reads an environment variable and @stdin reads standard input",290            "  --output-dir <dir>      write there (created if needed); default: next to each source",291            "  --output <file>         exact output file, for a single job",292            "  --name-template <t>     tokens {name} {ext} {date} {counter} {counter:3} {width} {height}; default {name}",293            "  --on-conflict <policy>  rename (default: 'name 2.ext'), skip, replace, or fail. Nothing is ever overwritten",294            "                          without 'replace'; a source file is never replaced",295            "  --jobs <n>              conversions running at once (default: performance cores, at most 8)",296            "  --recursive             convert the files inside the given folders (hidden files and links skipped)",297            "  --combine               one job from all the files, for paths taking several sources (images → PDF);",298            "                          implied by --operation when that operation takes several sources",299            "  --from <format>         do not detect: treat the sources as this format",300            "  --engine <id>           force one engine's direct path",301            "  --no-chain              refuse two-step plans",302            "  --json                  print one JSON document (summary + every job) instead of text",303            "  --quiet                 print only failures",304            "  --verbose               print every validation check and measured fact",305            "  --no-history            do not open or write the history database",306        ])307308    struct Plan {309        var requests: [ConversionJobRequest]310        var configuration: ConversionService.Configuration311        var failOnConflict: Bool312        var display: BatchRunner.Display313    }314315    /// Everything decided from the command line, before anything runs.316    static func prepare(_ arguments: ParsedArguments, io: CommandIO, registry: FormatRegistry) throws -> Plan {317        guard let toRaw = arguments.value("to") else { throw UsageError("convert needs --to <format>.") }318        let destinationFormat = try Shared.format(toRaw, option: "--to")319        let sourceFormat = try arguments.value("from").map { try Shared.format($0, option: "--from") }320        let operation = arguments.value("operation")321        if let engine = arguments.value("engine"), registry.engine(id: engine) == nil {322            throw UsageError("--engine: no engine '\(engine)'. Engines: \(registry.engines.map(\.id).joined(separator: ", ")).")323        }324        if arguments.value("output") != nil, arguments.value("output-dir") != nil { throw UsageError("Give --output or --output-dir, not both.") }325        if arguments.value("output") != nil, arguments.value("name-template") != nil { throw UsageError("--output already names the file; --name-template cannot be combined with it.") }326        if arguments.flag("json"), arguments.flag("quiet") { throw UsageError("Give --json or --quiet, not both.") }327328        var policy = ConflictPolicy.rename329        var failOnConflict = false330        if let raw = arguments.value("on-conflict") {331            switch raw {332            case "rename": policy = .rename333            case "skip": policy = .skip334            case "replace": policy = .replace335            case "fail": policy = .askUser; failOnConflict = true336            default: throw UsageError("--on-conflict expects rename, skip, replace or fail, not '\(raw)'.")337            }338        }339        let jobs = try arguments.int("jobs", in: 1...64)340341        // Inputs.342        var sources: [URL] = []343        for raw in arguments.positionals {344            let url = Shared.fileURL(raw)345            var isDirectory: ObjCBool = false346            if arguments.flag("recursive"), FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory), isDirectory.boolValue {347                sources += filesInside(url)348            } else {349                sources.append(url)350            }351        }352        guard !sources.isEmpty else { throw CommandFailure(.inputError, "No file to convert (the folders given are empty).") }353354        // Options: secrets are resolved once; types come from the descriptors of each file's own path.355        let raws = try OptionParsing.resolve(arguments.all("option"), io: io)356        var cache: [String: ConversionOptions] = [:]357        var unknownKeys = Set<String>()358        let planner = ConversionPlanner(registry: registry)359        func options(for url: URL) throws -> ConversionOptions {360            guard !raws.isEmpty else { return ConversionOptions() }361            let format = sourceFormat ?? FileTypeDetector.detect(url).format362            let key = format?.rawValue ?? ""363            if let cached = cache[key] { return cached }364            var descriptors: [OptionDescriptor] = []365            var known = false366            if let format, let plan = try? planner.plan(from: format, to: destinationFormat, operation: operation, engine: arguments.value("engine"),367                                                        allowChaining: !arguments.flag("no-chain")),368               let last = plan.steps.last, let engine = registry.engine(id: last.engine) {369                descriptors = engine.options(for: last)370                known = true371            }372            let typed = try OptionParsing.typed(raws, descriptors: descriptors) { key in if known { unknownKeys.insert(key) } }373            cache[key] = typed374            return typed375        }376377        // One job per file, or one job for all.378        var combine = arguments.flag("combine")379        if sources.count > 1 {380            let first = sourceFormat ?? FileTypeDetector.detect(sources[0]).format381            let multi = first.map { registry.paths(from: $0, to: destinationFormat, operation: .some(operation), engine: arguments.value("engine")).contains(where: \.acceptsMultipleSources) } ?? false382            if combine, !multi { throw UsageError("--combine: this conversion takes one file at a time.") }383            if operation != nil, multi { combine = true }384        } else {385            combine = false386        }387388        var directory = try arguments.value("output-dir").map { raw -> URL in389            let url = Shared.fileURL(raw)390            var isDirectory: ObjCBool = false391            if FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory), !isDirectory.boolValue {392                throw UsageError("--output-dir: \(url.path) is a file, not a folder.")393            }394            return url395        }396        var template = arguments.value("name-template") ?? OutputNameTemplate.default397        if let raw = arguments.value("output") {398            guard combine || sources.count == 1 else { throw UsageError("--output names one file but \(sources.count) files were given. Use --output-dir.") }399            let url = Shared.fileURL(raw)400            directory = url.deletingLastPathComponent()401            let format = FormatCatalog.format(destinationFormat)402            if destinationFormat == .folder || format?.traits.contains(.directory) == true {403                template = url.lastPathComponent404            } else {405                let preferred = format?.preferredExtension ?? destinationFormat.rawValue406                let name = url.lastPathComponent407                guard name.lowercased().hasSuffix("." + preferred), name.count > preferred.count + 1 else {408                    throw UsageError("--output: a \(destinationFormat.rawValue) file is written with the extension .\(preferred); name it '\(url.deletingPathExtension().lastPathComponent).\(preferred)'.")409                }410                template = String(name.dropLast(preferred.count + 1))411            }412        }413        let destination = OutputDestination(directory: directory, nameTemplate: template, conflictPolicy: policy)414        let history = !arguments.flag("no-history")415        let batch = UUID()416        var requests: [ConversionJobRequest] = []417        if combine {418            requests.append(ConversionJobRequest(sources: sources, sourceFormat: sourceFormat, destinationFormat: destinationFormat, operation: operation,419                                                 options: try options(for: sources[0]), engineID: arguments.value("engine"), destination: destination,420                                                 batchID: batch, counter: 1, allowChaining: !arguments.flag("no-chain"), recordInHistory: history))421        } else {422            for (index, url) in sources.enumerated() {423                requests.append(ConversionJobRequest(sources: [url], sourceFormat: sourceFormat, destinationFormat: destinationFormat, operation: operation,424                                                     options: try options(for: url), engineID: arguments.value("engine"), destination: destination,425                                                     batchID: batch, counter: index + 1, allowChaining: !arguments.flag("no-chain"), recordInHistory: history))426            }427        }428        for key in unknownKeys.sorted() {429            io.err("orix-convert: warning: '\(key)' is not a documented setting of this conversion; it is passed as given.")430        }431        return Plan(requests: requests, configuration: ConversionService.Configuration(maxConcurrentJobs: jobs, recordHistory: history),432                    failOnConflict: failOnConflict,433                    display: BatchRunner.Display(quiet: arguments.flag("quiet"), verbose: arguments.flag("verbose"), json: arguments.flag("json")))434    }435436    /// Regular files inside a folder, sorted; hidden items and symbolic links are skipped, packages are not entered.437    static func filesInside(_ directory: URL) -> [URL] {438        guard let enumerator = FileManager.default.enumerator(at: directory, includingPropertiesForKeys: [.isRegularFileKey, .isSymbolicLinkKey],439                                                              options: [.skipsHiddenFiles, .skipsPackageDescendants]) else { return [] }440        var files: [URL] = []441        for case let url as URL in enumerator {442            let values = try? url.resourceValues(forKeys: [.isRegularFileKey, .isSymbolicLinkKey])443            if values?.isSymbolicLink == true { continue }444            if values?.isRegularFile == true { files.append(url.standardizedFileURL) }445        }446        return files.sorted { $0.path < $1.path }447    }448449    static func run(_ arguments: ParsedArguments, io: CommandIO) async throws -> ExitCode {450        try await run(arguments, io: io, configure: nil, afterEnqueue: nil).code451    }452453    /// Testable form: `configure` adjusts the service configuration (work root, history file), `afterEnqueue` runs once454    /// the jobs are in the queue.455    static func run(_ arguments: ParsedArguments, io: CommandIO, configure: ((inout ConversionService.Configuration) -> Void)?,456                    afterEnqueue: (@Sendable (ConversionService, [UUID]) async -> Void)?) async throws -> (code: ExitCode, report: BatchReport) {457        var probe = ConversionService.Configuration(recordHistory: false)458        configure?(&probe)459        let registry = FormatRegistry(engines: probe.engines ?? ORIXEngines.all())460        var plan = try prepare(arguments, io: io, registry: registry)461        configure?(&plan.configuration)462        let service = ConversionService(configuration: plan.configuration)463        if let error = service.historyError, !plan.display.quiet {464            io.err("orix-convert: warning: the history database could not be opened (\(error)); conversions are not recorded.")465        }466        let runner = BatchRunner(service: service, io: io, display: plan.display, failOnConflict: plan.failOnConflict)467        let report = await runner.run(plan.requests) { ids in await afterEnqueue?(service, ids) }468        if plan.display.json {469            io.out(try Shared.json(report))470        } else if !plan.display.quiet || report.summary.failed > 0 || report.summary.cancelled > 0 {471            io.out(BatchRunner.summaryLine(report.summary))472        }473        if !report.leftoverTemporaryItems.isEmpty {474            io.err("orix-convert: warning: temporary folders were left behind in \(service.workDirectories.root.path): \(report.leftoverTemporaryItems.joined(separator: ", "))")475        }476        return (ExitCode(rawValue: report.summary.exitCode) ?? .conversionFailed, report)477    }478}479
Files main
Menu

Theme