Repository Public License
1// ORIX Convert — Packages/ConvertKit/Sources/ConvertCLI/ConvertCommand.swift2// Description: `orix-convert convert`: turns files into queue jobs, follows them through the real queue (detector,3// planner, engine, validation, safe writer), prints measured progress, warnings, checks and a summary.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import ConvertCore7import ConvertKit89/// Result of one job, as printed and as exported in JSON.10public struct JobReport: Codable, Sendable {11 public struct Output: Codable, Sendable { public let path: String; public let outcome: String }12 public var sources: [String]13 public var sourceFormat: String?14 public var destinationFormat: String15 public var operation: String?16 public var state: String17 public var engines: [String]18 public var chained: Bool19 public var outputs: [Output]20 public var warnings: [ConversionWarning]21 public var checks: [ValidationCheck]22 public var facts: [String: String]23 public var error: String?24 public var errorKind: String?25 public var seconds: Double?26 public var bytesBefore: Int64?27 public var bytesAfter: Int64?28}2930public struct BatchSummary: Codable, Sendable {31 public var total = 0, finished = 0, failed = 0, skipped = 0, cancelled = 032 public var bytesBefore: Int64 = 0, bytesAfter: Int64 = 033 public var seconds = 0.034 public var interrupted = false35 public var exitCode: Int32 = 036}3738public struct BatchReport: Codable, Sendable {39 public var summary: BatchSummary40 public var jobs: [JobReport]41 /// Work folders of this process still present after the run (must be empty).42 public var leftoverTemporaryItems: [String]43}4445/// Runs a batch of job requests on a service's queue and reports as jobs settle.46struct BatchRunner: Sendable {47 struct Display: Sendable {48 var quiet = false49 var verbose = false50 var json = false51 }5253 let service: ConversionService54 let io: CommandIO55 let display: Display56 /// `--on-conflict fail`: jobs run with `askUser`; a job that stops on an existing destination is cancelled and57 /// reported as failed.58 let failOnConflict: Bool5960 func run(_ requests: [ConversionJobRequest], afterEnqueue: (@Sendable ([UUID]) async -> Void)? = nil) async -> BatchReport {61 let started = Date()62 let screen = Screen(io: io, showsProgress: !display.quiet && !display.json)63 let queue = service.queue64 let total = requests.count65 let tracker = ProgressTracker(total: total)6667 let events = await queue.events()68 let observer = Task {69 for await event in events {70 switch event {71 case .jobChanged(let job):72 tracker.update(job.id, name: job.primarySource?.lastPathComponent ?? "", state: job.state, fraction: job.progress.fraction, eta: job.estimatedSecondsRemaining)73 case .jobProgress(let id, let progress, let eta):74 tracker.progress(id, fraction: progress.fraction, eta: eta, step: progress.stepIndex, steps: progress.stepCount)75 default:76 continue77 }78 screen.progress(tracker.line())79 }80 }81 let interrupt = InterruptCenter.shared.register {82 Task { await queue.cancelAll() }83 }84 defer { InterruptCenter.shared.unregister(interrupt) }8586 let ids = await queue.enqueue(batch: requests)87 if let afterEnqueue { await afterEnqueue(ids) }8889 var reports = [JobReport?](repeating: nil, count: ids.count)90 await withTaskGroup(of: (Int, JobReport).self) { group in91 for (index, id) in ids.enumerated() {92 group.addTask { (index, await self.settle(id, request: requests[index])) }93 }94 for await (index, report) in group {95 reports[index] = report96 tracker.settled()97 if !display.json { print(report, index: index + 1, total: total, screen: screen) }98 }99 }100 observer.cancel()101 screen.clear()102103 var summary = BatchSummary()104 let jobs = reports.compactMap { $0 }105 summary.total = jobs.count106 for job in jobs {107 switch job.state {108 case JobState.finished.rawValue: summary.finished += 1109 case JobState.skipped.rawValue: summary.skipped += 1110 case JobState.cancelled.rawValue: summary.cancelled += 1111 default: summary.failed += 1112 }113 summary.bytesBefore += job.bytesBefore ?? 0114 summary.bytesAfter += job.bytesAfter ?? 0115 }116 summary.seconds = Date().timeIntervalSince(started)117 summary.interrupted = InterruptCenter.shared.isInterrupted118 summary.exitCode = Self.exitCode(jobs: jobs, interrupted: summary.interrupted).rawValue119 let prefix = "\(ProcessInfo.processInfo.processIdentifier)-"120 return BatchReport(summary: summary, jobs: jobs, leftoverTemporaryItems: service.workRootLeftovers().filter { $0.hasPrefix(prefix) })121 }122123 /// Waits for a job to settle and turns the outcome into a report.124 private func settle(_ id: UUID, request: ConversionJobRequest) async -> JobReport {125 let queue = service.queue126 var conflictMessage: String?127 var job = await queue.waitForJob(id)128 if let waiting = job, waiting.state == .awaitingDecision {129 let existing = waiting.conflict?.existing.path ?? "the destination"130 conflictMessage = "\(existing) already exists. Nothing was written (use --on-conflict rename, skip or replace)."131 await queue.cancel(id)132 job = await queue.waitForJob(id)133 }134 guard let job else {135 return JobReport(sources: request.sources.map(\.path), sourceFormat: nil, destinationFormat: request.destinationFormat.rawValue,136 operation: request.operation, state: JobState.failed.rawValue, engines: [], chained: false, outputs: [], warnings: [],137 checks: [], facts: [:], error: "The job disappeared from the queue.", errorKind: "internalError", seconds: nil,138 bytesBefore: nil, bytesAfter: nil)139 }140 var state = job.state.rawValue141 var error = job.error142 var kind = job.failure?.kind143 if let conflictMessage {144 state = JobState.failed.rawValue; error = conflictMessage; kind = "destinationExists"145 }146 if job.state == .skipped, error == nil {147 error = job.log.last(where: { $0.message.contains("exist") || $0.message.contains("Skipped") })?.message148 }149 return JobReport(sources: job.request.sources.map(\.path), sourceFormat: job.sourceFormat?.rawValue,150 destinationFormat: job.request.destinationFormat.rawValue, operation: job.request.operation, state: state,151 engines: job.plan?.engines ?? [], chained: job.plan?.isChained ?? false,152 outputs: job.outputs.map { JobReport.Output(path: $0.url.path, outcome: $0.outcome.rawValue) },153 warnings: job.warnings, checks: job.checks, facts: job.facts, error: error, errorKind: kind, seconds: job.duration,154 bytesBefore: job.bytesBefore, bytesAfter: job.bytesAfter)155 }156157 private func print(_ report: JobReport, index: Int, total: Int, screen: Screen) {158 let name = report.sources.first.map { ($0 as NSString).lastPathComponent } ?? "?"159 let more = report.sources.count > 1 ? " (+\(report.sources.count - 1) more)" : ""160 switch report.state {161 case JobState.finished.rawValue:162 guard !display.quiet else { return }163 var details: [String] = []164 if let source = report.sourceFormat { details.append("\(source) → \(report.destinationFormat)" + (report.operation.map { " [\($0)]" } ?? "")) }165 if !report.engines.isEmpty { details.append((report.chained ? "chained: " : "") + report.engines.joined(separator: " + ")) }166 if let before = report.bytesBefore, let after = report.bytesAfter { details.append("\(Shared.bytes(before)) → \(Shared.bytes(after))") }167 if let seconds = report.seconds { details.append(Shared.seconds(seconds)) }168 let written = report.outputs.filter { $0.outcome != "skipped" }169 let target = written.count == 1 ? written[0].path : "\(written.count) items in \((written.first?.path as NSString?)?.deletingLastPathComponent ?? "")"170 screen.line("ok \(name)\(more) → \(target) (\(details.joined(separator: ", ")))")171 for output in report.outputs where output.outcome != "written" {172 screen.line(" \(output.outcome): \(output.path)")173 }174 case JobState.skipped.rawValue:175 guard !display.quiet else { return }176 screen.line("skipped \(name)\(more): \(report.error ?? "the destination already exists.")")177 case JobState.cancelled.rawValue:178 screen.error("cancelled \(name)\(more)")179 default:180 screen.error("FAILED \(name)\(more): \(report.error ?? "unknown error")")181 }182 guard !display.quiet else { return }183 for warning in report.warnings {184 screen.line(" warning [\(warning.severity.rawValue)] \(warning.code): \(warning.message)")185 }186 if !report.checks.isEmpty {187 let failed = report.checks.filter { !$0.passed }188 if display.verbose || !failed.isEmpty {189 for check in report.checks where display.verbose || !check.passed {190 screen.line(" check \(check.passed ? "passed" : "FAILED") \(check.name)\(check.detail.isEmpty ? "" : ": \(check.detail)")")191 }192 } else {193 let names = Array(Set(report.checks.map(\.name))).sorted()194 screen.line(" validated: \(report.checks.count) check(s) passed (\(names.prefix(8).joined(separator: ", "))\(names.count > 8 ? ", …" : ""))")195 }196 }197 if display.verbose {198 for (key, value) in report.facts.sorted(by: { $0.key < $1.key }) { screen.line(" fact \(key) = \(value)") }199 }200 }201202 static func exitCode(jobs: [JobReport], interrupted: Bool) -> ExitCode {203 if interrupted { return .interrupted }204 let failures = jobs.filter { $0.state != JobState.finished.rawValue && $0.state != JobState.skipped.rawValue }205 guard !failures.isEmpty else { return .success }206 let kinds = Set(failures.map { $0.errorKind ?? "" })207 if kinds == ["unsupportedPath"] { return .unsupported }208 if kinds == ["destinationExists"] { return .destinationExists }209 if kinds.isSubset(of: ["unreadableInput", "malformedInput"]), failures.count == jobs.count { return .inputError }210 return .conversionFailed211 }212213 static func summaryLine(_ summary: BatchSummary) -> String {214 var parts = ["\(summary.finished) converted", "\(summary.failed) failed", "\(summary.skipped) skipped"]215 if summary.cancelled > 0 { parts.append("\(summary.cancelled) cancelled") }216 var line = "Summary: \(summary.total) job(s) — " + parts.joined(separator: ", ")217 if summary.finished > 0 { line += " — \(Shared.bytes(summary.bytesBefore)) in, \(Shared.bytes(summary.bytesAfter)) out" }218 return line + " — \(Shared.seconds(summary.seconds))" + (summary.interrupted ? " — interrupted" : "")219 }220}221222/// Live state behind the progress line.223final class ProgressTracker: @unchecked Sendable {224 private struct Active { var name: String; var fraction: Double?; var eta: Double?; var step = 0; var steps = 1; var state: JobState }225 private let lock = NSLock()226 private let total: Int227 private var done = 0228 private var active: [UUID: Active] = [:]229 private var order: [UUID] = []230231 init(total: Int) { self.total = total }232233 func update(_ id: UUID, name: String, state: JobState, fraction: Double?, eta: Double?) {234 lock.lock(); defer { lock.unlock() }235 if state.isActive {236 if active[id] == nil { order.append(id) }237 var entry = active[id] ?? Active(name: name, fraction: fraction, eta: eta, state: state)238 entry.state = state239 active[id] = entry240 } else {241 active[id] = nil242 order.removeAll { $0 == id }243 }244 }245246 func progress(_ id: UUID, fraction: Double?, eta: Double?, step: Int, steps: Int) {247 lock.lock(); defer { lock.unlock() }248 guard var entry = active[id] else { return }249 entry.fraction = fraction; entry.eta = eta; entry.step = step; entry.steps = steps250 active[id] = entry251 }252253 func settled() { lock.lock(); done += 1; lock.unlock() }254255 /// `[3/10] 2 running · photo.png 42 % · about 12 s left` — or `working (not measurable)` when the engine cannot256 /// measure its progress. A percentage is shown only when an engine reported one.257 func line() -> String {258 lock.lock(); defer { lock.unlock() }259 var text = "[\(done)/\(total)]"260 guard let first = order.first, let entry = active[first] else { return text + " waiting" }261 if active.count > 1 { text += " \(active.count) running ·" }262 text += " \(entry.name)"263 if entry.steps > 1 { text += " step \(entry.step + 1)/\(entry.steps)" }264 if entry.state == .validating {265 text += " verifying"266 } else if entry.state == .preparing {267 text += " preparing"268 } else if let fraction = entry.fraction {269 text += " \(Int((fraction * 100).rounded())) %"270 if let eta = entry.eta, eta >= 1 { text += " · about \(Int(eta.rounded())) s left" }271 } else {272 text += " working (progress not measurable)"273 }274 return text275 }276}277278enum ConvertCommand: Command {279 static let spec = CommandSpec(280 "convert", summary: "Convert files (real queue, validation, safe writer).",281 usage: "convert <files…> --to <format> [--operation <op>] [--option key=value …] [--output-dir <dir> | --output <file>] [--name-template <t>] [--on-conflict rename|skip|replace|fail] [--jobs <n>] [--recursive] [--combine] [--from <format>] [--engine <id>] [--json] [--quiet] [--verbose] [--no-history]",282 flags: ["json", "quiet", "verbose", "no-history", "recursive", "combine", "no-chain"],283 options: ["to", "operation", "output-dir", "output", "name-template", "on-conflict", "jobs", "from", "engine"],284 repeated: ["option"], minPositionals: 1, unbounded: true,285 details: [286 " --to <format> destination format (identifier or extension: jpeg, jpg, pdf, folder…)",287 " --operation <op> named operation (pdf.merge, image.vectorize, video.extract-audio…); see 'paths'",288 " --option key=value a setting of the path (see 'options'); repeatable. For a key containing 'password',",289 " the value @env:NAME reads an environment variable and @stdin reads standard input",290 " --output-dir <dir> write there (created if needed); default: next to each source",291 " --output <file> exact output file, for a single job",292 " --name-template <t> tokens {name} {ext} {date} {counter} {counter:3} {width} {height}; default {name}",293 " --on-conflict <policy> rename (default: 'name 2.ext'), skip, replace, or fail. Nothing is ever overwritten",294 " without 'replace'; a source file is never replaced",295 " --jobs <n> conversions running at once (default: performance cores, at most 8)",296 " --recursive convert the files inside the given folders (hidden files and links skipped)",297 " --combine one job from all the files, for paths taking several sources (images → PDF);",298 " implied by --operation when that operation takes several sources",299 " --from <format> do not detect: treat the sources as this format",300 " --engine <id> force one engine's direct path",301 " --no-chain refuse two-step plans",302 " --json print one JSON document (summary + every job) instead of text",303 " --quiet print only failures",304 " --verbose print every validation check and measured fact",305 " --no-history do not open or write the history database",306 ])307308 struct Plan {309 var requests: [ConversionJobRequest]310 var configuration: ConversionService.Configuration311 var failOnConflict: Bool312 var display: BatchRunner.Display313 }314315 /// Everything decided from the command line, before anything runs.316 static func prepare(_ arguments: ParsedArguments, io: CommandIO, registry: FormatRegistry) throws -> Plan {317 guard let toRaw = arguments.value("to") else { throw UsageError("convert needs --to <format>.") }318 let destinationFormat = try Shared.format(toRaw, option: "--to")319 let sourceFormat = try arguments.value("from").map { try Shared.format($0, option: "--from") }320 let operation = arguments.value("operation")321 if let engine = arguments.value("engine"), registry.engine(id: engine) == nil {322 throw UsageError("--engine: no engine '\(engine)'. Engines: \(registry.engines.map(\.id).joined(separator: ", ")).")323 }324 if arguments.value("output") != nil, arguments.value("output-dir") != nil { throw UsageError("Give --output or --output-dir, not both.") }325 if arguments.value("output") != nil, arguments.value("name-template") != nil { throw UsageError("--output already names the file; --name-template cannot be combined with it.") }326 if arguments.flag("json"), arguments.flag("quiet") { throw UsageError("Give --json or --quiet, not both.") }327328 var policy = ConflictPolicy.rename329 var failOnConflict = false330 if let raw = arguments.value("on-conflict") {331 switch raw {332 case "rename": policy = .rename333 case "skip": policy = .skip334 case "replace": policy = .replace335 case "fail": policy = .askUser; failOnConflict = true336 default: throw UsageError("--on-conflict expects rename, skip, replace or fail, not '\(raw)'.")337 }338 }339 let jobs = try arguments.int("jobs", in: 1...64)340341 // Inputs.342 var sources: [URL] = []343 for raw in arguments.positionals {344 let url = Shared.fileURL(raw)345 var isDirectory: ObjCBool = false346 if arguments.flag("recursive"), FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory), isDirectory.boolValue {347 sources += filesInside(url)348 } else {349 sources.append(url)350 }351 }352 guard !sources.isEmpty else { throw CommandFailure(.inputError, "No file to convert (the folders given are empty).") }353354 // Options: secrets are resolved once; types come from the descriptors of each file's own path.355 let raws = try OptionParsing.resolve(arguments.all("option"), io: io)356 var cache: [String: ConversionOptions] = [:]357 var unknownKeys = Set<String>()358 let planner = ConversionPlanner(registry: registry)359 func options(for url: URL) throws -> ConversionOptions {360 guard !raws.isEmpty else { return ConversionOptions() }361 let format = sourceFormat ?? FileTypeDetector.detect(url).format362 let key = format?.rawValue ?? ""363 if let cached = cache[key] { return cached }364 var descriptors: [OptionDescriptor] = []365 var known = false366 if let format, let plan = try? planner.plan(from: format, to: destinationFormat, operation: operation, engine: arguments.value("engine"),367 allowChaining: !arguments.flag("no-chain")),368 let last = plan.steps.last, let engine = registry.engine(id: last.engine) {369 descriptors = engine.options(for: last)370 known = true371 }372 let typed = try OptionParsing.typed(raws, descriptors: descriptors) { key in if known { unknownKeys.insert(key) } }373 cache[key] = typed374 return typed375 }376377 // One job per file, or one job for all.378 var combine = arguments.flag("combine")379 if sources.count > 1 {380 let first = sourceFormat ?? FileTypeDetector.detect(sources[0]).format381 let multi = first.map { registry.paths(from: $0, to: destinationFormat, operation: .some(operation), engine: arguments.value("engine")).contains(where: \.acceptsMultipleSources) } ?? false382 if combine, !multi { throw UsageError("--combine: this conversion takes one file at a time.") }383 if operation != nil, multi { combine = true }384 } else {385 combine = false386 }387388 var directory = try arguments.value("output-dir").map { raw -> URL in389 let url = Shared.fileURL(raw)390 var isDirectory: ObjCBool = false391 if FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory), !isDirectory.boolValue {392 throw UsageError("--output-dir: \(url.path) is a file, not a folder.")393 }394 return url395 }396 var template = arguments.value("name-template") ?? OutputNameTemplate.default397 if let raw = arguments.value("output") {398 guard combine || sources.count == 1 else { throw UsageError("--output names one file but \(sources.count) files were given. Use --output-dir.") }399 let url = Shared.fileURL(raw)400 directory = url.deletingLastPathComponent()401 let format = FormatCatalog.format(destinationFormat)402 if destinationFormat == .folder || format?.traits.contains(.directory) == true {403 template = url.lastPathComponent404 } else {405 let preferred = format?.preferredExtension ?? destinationFormat.rawValue406 let name = url.lastPathComponent407 guard name.lowercased().hasSuffix("." + preferred), name.count > preferred.count + 1 else {408 throw UsageError("--output: a \(destinationFormat.rawValue) file is written with the extension .\(preferred); name it '\(url.deletingPathExtension().lastPathComponent).\(preferred)'.")409 }410 template = String(name.dropLast(preferred.count + 1))411 }412 }413 let destination = OutputDestination(directory: directory, nameTemplate: template, conflictPolicy: policy)414 let history = !arguments.flag("no-history")415 let batch = UUID()416 var requests: [ConversionJobRequest] = []417 if combine {418 requests.append(ConversionJobRequest(sources: sources, sourceFormat: sourceFormat, destinationFormat: destinationFormat, operation: operation,419 options: try options(for: sources[0]), engineID: arguments.value("engine"), destination: destination,420 batchID: batch, counter: 1, allowChaining: !arguments.flag("no-chain"), recordInHistory: history))421 } else {422 for (index, url) in sources.enumerated() {423 requests.append(ConversionJobRequest(sources: [url], sourceFormat: sourceFormat, destinationFormat: destinationFormat, operation: operation,424 options: try options(for: url), engineID: arguments.value("engine"), destination: destination,425 batchID: batch, counter: index + 1, allowChaining: !arguments.flag("no-chain"), recordInHistory: history))426 }427 }428 for key in unknownKeys.sorted() {429 io.err("orix-convert: warning: '\(key)' is not a documented setting of this conversion; it is passed as given.")430 }431 return Plan(requests: requests, configuration: ConversionService.Configuration(maxConcurrentJobs: jobs, recordHistory: history),432 failOnConflict: failOnConflict,433 display: BatchRunner.Display(quiet: arguments.flag("quiet"), verbose: arguments.flag("verbose"), json: arguments.flag("json")))434 }435436 /// Regular files inside a folder, sorted; hidden items and symbolic links are skipped, packages are not entered.437 static func filesInside(_ directory: URL) -> [URL] {438 guard let enumerator = FileManager.default.enumerator(at: directory, includingPropertiesForKeys: [.isRegularFileKey, .isSymbolicLinkKey],439 options: [.skipsHiddenFiles, .skipsPackageDescendants]) else { return [] }440 var files: [URL] = []441 for case let url as URL in enumerator {442 let values = try? url.resourceValues(forKeys: [.isRegularFileKey, .isSymbolicLinkKey])443 if values?.isSymbolicLink == true { continue }444 if values?.isRegularFile == true { files.append(url.standardizedFileURL) }445 }446 return files.sorted { $0.path < $1.path }447 }448449 static func run(_ arguments: ParsedArguments, io: CommandIO) async throws -> ExitCode {450 try await run(arguments, io: io, configure: nil, afterEnqueue: nil).code451 }452453 /// Testable form: `configure` adjusts the service configuration (work root, history file), `afterEnqueue` runs once454 /// the jobs are in the queue.455 static func run(_ arguments: ParsedArguments, io: CommandIO, configure: ((inout ConversionService.Configuration) -> Void)?,456 afterEnqueue: (@Sendable (ConversionService, [UUID]) async -> Void)?) async throws -> (code: ExitCode, report: BatchReport) {457 var probe = ConversionService.Configuration(recordHistory: false)458 configure?(&probe)459 let registry = FormatRegistry(engines: probe.engines ?? ORIXEngines.all())460 var plan = try prepare(arguments, io: io, registry: registry)461 configure?(&plan.configuration)462 let service = ConversionService(configuration: plan.configuration)463 if let error = service.historyError, !plan.display.quiet {464 io.err("orix-convert: warning: the history database could not be opened (\(error)); conversions are not recorded.")465 }466 let runner = BatchRunner(service: service, io: io, display: plan.display, failOnConflict: plan.failOnConflict)467 let report = await runner.run(plan.requests) { ids in await afterEnqueue?(service, ids) }468 if plan.display.json {469 io.out(try Shared.json(report))470 } else if !plan.display.quiet || report.summary.failed > 0 || report.summary.cancelled > 0 {471 io.out(BatchRunner.summaryLine(report.summary))472 }473 if !report.leftoverTemporaryItems.isEmpty {474 io.err("orix-convert: warning: temporary folders were left behind in \(service.workDirectories.root.path): \(report.leftoverTemporaryItems.joined(separator: ", "))")475 }476 return (ExitCode(rawValue: report.summary.exitCode) ?? .conversionFailed, report)477 }478}479