Repository Public License
1// ORIX Convert — Packages/ConvertKit/Sources/ConvertCLI/Stress.swift2// Description: Mixed-batch robustness run (`orix-convert dev stress`): 300 files of mixed generated formats, garbage,3// empty files, a file deleted mid-run, a read-only destination and name collisions, all through the4// real `convert` command; checks every outcome against what the registry says, and that nothing leaks.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import UniformTypeIdentifiers8import ConvertCore9import ConvertKit1011enum Stress {12 struct Check: Codable { let name: String; let passed: Bool; let detail: String }13 struct Scenario: Codable {14 let name: String15 let command: String16 let exitCode: Int3217 let summary: BatchSummary18 let failuresByKind: [String: Int]19 }20 struct Report: Codable {21 let date: String, machine: String22 let loadAverage: [Double]23 let files: [String: Int]24 let scenarios: [Scenario]25 let checks: [Check]26 let unexpected: [String]27 let temporaryRoot: String28 let temporaryRootEntries: [String]29 let seconds: Double30 }3132 final class Buffer: @unchecked Sendable {33 private let lock = NSLock()34 private var lines: [String] = []35 func add(_ line: String) { lock.lock(); lines.append(line); lock.unlock() }36 var text: String { lock.lock(); defer { lock.unlock() }; return lines.joined(separator: "\n") }37 }3839 static func run(_ arguments: ParsedArguments, io: CommandIO) async throws -> ExitCode {40 let started = Date()41 let fm = FileManager.default42 let work = arguments.value("work-dir").map(Shared.fileURL) ?? fm.temporaryDirectory.appendingPathComponent("orix-convert-stress-\(ProcessInfo.processInfo.processIdentifier)", isDirectory: true)43 guard let reportDirectory = arguments.value("report-dir").map(Shared.fileURL) ?? Repository.root()?.appendingPathComponent("docs/reports") else {44 throw UsageError("dev stress needs --report-dir when it does not run from the repository.")45 }46 let jobs = try arguments.int("jobs", in: 1...32) ?? 647 try? fm.removeItem(at: work)48 let input = work.appendingPathComponent("in", isDirectory: true)49 try fm.createDirectory(at: input, withIntermediateDirectories: true)50 defer { if !arguments.flag("keep") { _ = chmod(work.appendingPathComponent("read-only").path, 0o755); try? fm.removeItem(at: work) } }5152 // 1. Material: 300 files.53 enum Kind: String { case valid, garbage, empty, collision, deleted }54 var files: [(url: URL, kind: Kind)] = []55 func add(_ name: String, _ data: Data, _ kind: Kind = .valid, folder: String? = nil) throws {56 let directory = folder.map { input.appendingPathComponent($0, isDirectory: true) } ?? input57 try fm.createDirectory(at: directory, withIntermediateDirectories: true)58 let url = directory.appendingPathComponent(name)59 try data.write(to: url)60 files.append((url, kind))61 }62 func png(_ seed: Int) throws -> Data {63 let url = work.appendingPathComponent("scratch.png")64 try Seeds.writeImage(try Seeds.image(width: 160 + seed % 5 * 32, height: 120 + seed % 3 * 24, seed: seed), to: url)65 return try Data(contentsOf: url)66 }67 func wav() throws -> Data {68 let url = work.appendingPathComponent("scratch.wav")69 try Seeds.writeWAV(to: url, seconds: 0.3)70 return try Data(contentsOf: url)71 }72 let wavData = try wav()73 for i in 1...60 { try add(String(format: "photo-%03d.png", i), try png(i)) }74 for i in 1...30 { try add(String(format: "sound-%03d.wav", i), wavData) }75 for i in 1...30 { try add(String(format: "note-%03d.txt", i), Data((Seeds.text + "\nNote \(i).\n").utf8)) }76 for i in 1...30 { try add(String(format: "page-%03d.md", i), Data((Seeds.markdown + "\n\nPage \(i).\n").utf8)) }77 for i in 1...30 { try add(String(format: "table-%03d.csv", i), Data(Seeds.csv(rows: 5 + i).utf8)) }78 for i in 1...30 { try add(String(format: "records-%03d.json", i), Data(Seeds.json(rows: 3 + i).utf8)) }79 for i in 1...20 { try add(String(format: "web-%03d.html", i), Data(Seeds.html.utf8)) }80 for i in 1...20 { try add(String(format: "drawing-%03d.svg", i), Data(Seeds.svg.utf8)) }81 var state: UInt64 = 4282 func noise(_ count: Int) -> Data {83 var bytes = [UInt8](repeating: 0, count: count)84 for i in bytes.indices { state = state &* 6364136223846793005 &+ 1442695040888963407; bytes[i] = UInt8(truncatingIfNeeded: state >> 56) }85 bytes[0] = 0; bytes[1] = 0xFF; bytes[2] = 086 return Data(bytes)87 }88 let garbageExtensions = ["png", "pdf", "docx", "mp3", "zip", "jpg", "xlsx", "mp4", "bin", "dat"]89 for i in 1...20 { try add(String(format: "garbage-%03d.%@", i, garbageExtensions[i % garbageExtensions.count]), noise(2000 + i * 37), .garbage) }90 let emptyExtensions = ["png", "txt", "pdf", "csv", "json"]91 for i in 1...15 { try add(String(format: "empty-%03d.%@", i, emptyExtensions[i % emptyExtensions.count]), Data(), .empty) }92 for i in 1...14 { try add(String(format: "photo-%03d.png", i % 7 + 1), try png(100 + i), .collision, folder: "dup-\(i)") }93 try add("vanishing.png", try png(999), .deleted)94 try? fm.removeItem(at: work.appendingPathComponent("scratch.png"))95 try? fm.removeItem(at: work.appendingPathComponent("scratch.wav"))96 let counts = Dictionary(grouping: files, by: { $0.kind.rawValue }).mapValues(\.count)97 io.err("Mixed batch: \(files.count) files (\(counts.sorted { $0.key < $1.key }.map { "\($0.value) \($0.key)" }.joined(separator: ", "))).")9899 var checks: [Check] = []100 var unexpected: [String] = []101 var scenarios: [Scenario] = []102 func check(_ name: String, _ passed: Bool, _ detail: String) {103 checks.append(Check(name: name, passed: passed, detail: detail))104 io.err(" \(passed ? "ok " : "FAIL") \(name): \(detail)")105 }106 func convert(_ name: String, _ argv: [String], afterEnqueue: (@Sendable (ConversionService, [UUID]) async -> Void)? = nil) async throws -> BatchReport {107 let out = Buffer(), err = Buffer()108 let captured = CommandIO(out: out.add, err: err.add, environment: io.environment)109 let parsed = try ArgumentParser.parse(argv, spec: ConvertCommand.spec)110 let (code, report) = try await ConvertCommand.run(parsed, io: captured, configure: nil, afterEnqueue: afterEnqueue)111 // The JSON the command printed must decode to the same summary: that is what a script would read.112 if let data = out.text.data(using: .utf8), let decoded = try? JSONDecoder().decode(BatchReport.self, from: data) {113 check("\(name): JSON summary matches the jobs", decoded.summary.total == decoded.jobs.count114 && decoded.summary.finished == decoded.jobs.filter { $0.state == "finished" }.count115 && decoded.summary.failed == decoded.jobs.filter { $0.state == "failed" }.count116 && decoded.summary.skipped == decoded.jobs.filter { $0.state == "skipped" }.count117 && decoded.summary.finished + decoded.summary.failed + decoded.summary.skipped + decoded.summary.cancelled == decoded.summary.total,118 "\(decoded.summary.total) jobs: \(decoded.summary.finished) finished, \(decoded.summary.failed) failed, \(decoded.summary.skipped) skipped, \(decoded.summary.cancelled) cancelled")119 } else {120 check("\(name): JSON output decodes", false, "the command did not print a decodable report")121 }122 let kinds = Dictionary(grouping: report.jobs.filter { $0.state == "failed" }, by: { $0.errorKind ?? "?" }).mapValues(\.count)123 scenarios.append(Scenario(name: name, command: "orix-convert convert " + argv.map { $0.hasPrefix(work.path) ? "<work>" + $0.dropFirst(work.path.count) : $0 }.suffix(12).joined(separator: " "),124 exitCode: code.rawValue, summary: report.summary, failuresByKind: kinds))125 check("\(name): no work folder left", report.leftoverTemporaryItems.isEmpty, report.leftoverTemporaryItems.isEmpty ? "none" : report.leftoverTemporaryItems.joined(separator: ", "))126 return report127 }128 func partials(in directory: URL) -> [String] {129 guard let enumerator = fm.enumerator(at: directory, includingPropertiesForKeys: nil, options: []) else { return [] }130 var found: [String] = []131 for case let url as URL in enumerator where url.lastPathComponent.contains(".partial") || url.lastPathComponent.hasPrefix(".orix-convert") { found.append(url.lastPathComponent) }132 return found133 }134135 // 2. Scenario A: everything to PDF, one output folder, names colliding, one file deleted while queued.136 let registry = Shared.registry()137 let planner = ConversionPlanner(registry: registry)138 var expected: [String: String] = [:]139 for file in files {140 switch file.kind {141 case .garbage, .deleted: expected[file.url.path] = "failed"142 case .empty:143 // A zero-byte .txt is a valid, empty text document (it becomes a blank page); every other empty file is unreadable.144 expected[file.url.path] = file.url.pathExtension == "txt" ? "finished" : "failed"145 case .valid, .collision:146 let format = FileTypeDetector.detect(file.url).format147 expected[file.url.path] = format.flatMap { try? planner.plan(from: $0, to: .pdf) } == nil ? "failed" : "finished"148 }149 }150 let outA = work.appendingPathComponent("out-a", isDirectory: true)151 let vanishing = files.last { $0.kind == .deleted }?.url152 let reportA = try await convert("mixed batch to PDF", files.map(\.url.path) + ["--to", "pdf", "--output-dir", outA.path, "--jobs", String(jobs), "--json", "--no-history"]) { _, _ in153 if let vanishing { try? FileManager.default.removeItem(at: vanishing) }154 }155 var mismatches = 0156 for job in reportA.jobs {157 guard let source = job.sources.first else { continue }158 let want = expected[source] ?? "?"159 if job.state != want {160 mismatches += 1161 unexpected.append("\((source as NSString).lastPathComponent): expected \(want), got \(job.state)\(job.error.map { " — \($0)" } ?? "")")162 }163 }164 check("mixed batch: every outcome is the one the registry predicts", mismatches == 0, "\(reportA.jobs.count) jobs, \(mismatches) unexpected outcome(s)")165 check("mixed batch: one job per file", reportA.jobs.count == files.count, "\(reportA.jobs.count) jobs for \(files.count) files")166 let expectedFinished = expected.values.filter { $0 == "finished" }.count167 check("mixed batch: summary counts", reportA.summary.finished == expectedFinished && reportA.summary.failed == files.count - expectedFinished && reportA.summary.skipped == 0,168 "finished \(reportA.summary.finished) (expected \(expectedFinished)), failed \(reportA.summary.failed) (expected \(files.count - expectedFinished))")169 check("mixed batch: exit status 1 (some conversions failed)", reportA.summary.exitCode == ExitCode.conversionFailed.rawValue, "exit \(reportA.summary.exitCode)")170 let writtenA = reportA.jobs.flatMap(\.outputs).map(\.path)171 check("mixed batch: every reported output exists and no two jobs share one", writtenA.allSatisfy { fm.fileExists(atPath: $0) } && Set(writtenA).count == writtenA.count,172 "\(writtenA.count) outputs, \(Set(writtenA).count) distinct")173 let renamed = reportA.jobs.flatMap(\.outputs).filter { $0.outcome == "renamed" }.count174 let collidingValid = files.filter { $0.kind == .collision }.count175 check("mixed batch: name collisions were numbered, not overwritten", renamed == collidingValid, "\(renamed) outputs renamed for \(collidingValid) colliding names")176 let garbageJobs = reportA.jobs.filter { ($0.sources.first.map { ($0 as NSString).lastPathComponent } ?? "").hasPrefix("garbage-") || ($0.sources.first.map { ($0 as NSString).lastPathComponent } ?? "").hasPrefix("empty-") }177 let emptyText = garbageJobs.filter { ($0.sources.first ?? "").hasSuffix(".txt") }178 check("mixed batch: an empty .txt is converted as an empty document", emptyText.allSatisfy { $0.state == "finished" }, "\(emptyText.count) empty .txt file(s): \(emptyText.filter { $0.state == "finished" }.count) converted to a blank PDF")179 check("mixed batch: garbage and the other empty files fail with a message and write nothing",180 garbageJobs.filter { !($0.sources.first ?? "").hasSuffix(".txt") }.allSatisfy { $0.state == "failed" && !($0.error ?? "").isEmpty && $0.outputs.isEmpty },181 "\(garbageJobs.count) files; kinds: \(Dictionary(grouping: garbageJobs, by: { $0.errorKind ?? "?" }).mapValues(\.count).sorted { $0.key < $1.key }.map { "\($0.key) \($0.value)" }.joined(separator: ", "))")182 if let gone = reportA.jobs.first(where: { $0.sources.first == vanishing?.path }) {183 check("mixed batch: the file deleted while queued fails cleanly", gone.state == "failed" && gone.errorKind == "unreadableInput", gone.error ?? gone.state)184 }185 check("mixed batch: no partial file in the output folder", partials(in: outA).isEmpty, partials(in: outA).isEmpty ? "none" : partials(in: outA).joined(separator: ", "))186 let sourcesIntact = files.filter { $0.kind != .deleted }.allSatisfy { fm.fileExists(atPath: $0.url.path) }187 check("mixed batch: no source file was touched", sourcesIntact, sourcesIntact ? "all \(files.count - 1) remaining sources present" : "a source is missing")188189 // 3. Scenario B: read-only destination.190 let readOnly = work.appendingPathComponent("read-only", isDirectory: true)191 try fm.createDirectory(at: readOnly, withIntermediateDirectories: true)192 _ = chmod(readOnly.path, 0o555)193 let ten = files.filter { $0.kind == .valid && $0.url.pathExtension == "png" }.prefix(10).map(\.url.path)194 let reportB = try await convert("read-only destination", ten + ["--to", "jpeg", "--output-dir", readOnly.path, "--jobs", String(jobs), "--json", "--no-history"])195 check("read-only destination: every job fails with a write error", reportB.summary.failed == 10 && reportB.jobs.allSatisfy { $0.errorKind == "writeFailed" },196 "failed \(reportB.summary.failed)/10; kinds: \(Set(reportB.jobs.compactMap(\.errorKind)).sorted().joined(separator: ", "))")197 _ = chmod(readOnly.path, 0o755)198 let leftB = (try? fm.contentsOfDirectory(atPath: readOnly.path)) ?? []199 check("read-only destination: nothing was written there", leftB.isEmpty, leftB.isEmpty ? "empty" : leftB.joined(separator: ", "))200201 // 4. Scenario C: an existing destination under each conflict policy.202 let outC = work.appendingPathComponent("out-c", isDirectory: true)203 let first = try await convert("conflicts: first write", ten + ["--to", "jpeg", "--output-dir", outC.path, "--json", "--no-history"])204 check("conflicts: first write succeeds", first.summary.finished == 10, "finished \(first.summary.finished)/10")205 func snapshot() -> [String: Data] {206 var result: [String: Data] = [:]207 for name in (try? fm.contentsOfDirectory(atPath: outC.path)) ?? [] { result[name] = try? Data(contentsOf: outC.appendingPathComponent(name)) }208 return result209 }210 let original = snapshot()211 let failRun = try await convert("conflicts: --on-conflict fail", ten + ["--to", "jpeg", "--output-dir", outC.path, "--on-conflict", "fail", "--json", "--no-history"])212 check("conflicts: fail → exit 5, nothing written or changed", failRun.summary.exitCode == ExitCode.destinationExists.rawValue && failRun.summary.failed == 10 && snapshot() == original,213 "exit \(failRun.summary.exitCode), failed \(failRun.summary.failed), folder unchanged: \(snapshot() == original)")214 let skipRun = try await convert("conflicts: --on-conflict skip", ten + ["--to", "jpeg", "--output-dir", outC.path, "--on-conflict", "skip", "--json", "--no-history"])215 check("conflicts: skip → exit 0, 10 skipped, nothing changed", skipRun.summary.exitCode == 0 && skipRun.summary.skipped == 10 && snapshot() == original,216 "exit \(skipRun.summary.exitCode), skipped \(skipRun.summary.skipped), folder unchanged: \(snapshot() == original)")217 let renameRun = try await convert("conflicts: default (rename)", ten + ["--to", "jpeg", "--output-dir", outC.path, "--json", "--no-history"])218 let afterRename = snapshot()219 check("conflicts: rename → 10 new numbered files, originals intact", renameRun.summary.finished == 10 && afterRename.count == 20 && original.allSatisfy { afterRename[$0.key] == $0.value },220 "finished \(renameRun.summary.finished), \(afterRename.count) files in the folder")221 let replaceRun = try await convert("conflicts: --on-conflict replace", ten + ["--to", "jpeg", "--option", "quality=30", "--output-dir", outC.path, "--on-conflict", "replace", "--json", "--no-history"])222 let afterReplace = snapshot()223 let replaced = original.keys.filter { afterReplace[$0] != original[$0] }.count224 check("conflicts: replace → the 10 files are replaced, no extra file", replaceRun.summary.finished == 10 && afterReplace.count == 20 && replaced == 10,225 "finished \(replaceRun.summary.finished), replaced \(replaced), \(afterReplace.count) files")226 check("conflicts: no partial file left", partials(in: outC).isEmpty, partials(in: outC).isEmpty ? "none" : partials(in: outC).joined(separator: ", "))227 let sourceReplace = try await convert("source protection", [ten[0], "--to", "png", "--on-conflict", "replace", "--json", "--no-history"])228 check("source protection: converting a file onto itself with replace is refused", sourceReplace.summary.failed == 1 && sourceReplace.jobs.first?.errorKind == "destinationExists" && fm.fileExists(atPath: ten[0]),229 sourceReplace.jobs.first?.error ?? "state \(sourceReplace.jobs.first?.state ?? "?")")230231 // 5. The application's temporary root.232 let root = AppPaths.temporaryRoot233 let entries = ((try? fm.contentsOfDirectory(atPath: root.path)) ?? []).sorted()234 let ours = entries.filter { $0.hasPrefix("\(ProcessInfo.processInfo.processIdentifier)-") }235 check("temporary root holds nothing of this run", ours.isEmpty, "\(root.path): \(entries.count) entr\(entries.count == 1 ? "y" : "ies") in total, \(ours.count) from this process")236237 let report = Report(date: ISO8601DateFormatter().string(from: started), machine: Sweep.machine(), loadAverage: Sweep.loadAverage(), files: counts,238 scenarios: scenarios, checks: checks, unexpected: unexpected, temporaryRoot: root.path, temporaryRootEntries: entries,239 seconds: Date().timeIntervalSince(started))240 try fm.createDirectory(at: reportDirectory, withIntermediateDirectories: true)241 try Data((try Shared.json(report) + "\n").utf8).write(to: reportDirectory.appendingPathComponent("mixed-batch.json"), options: .atomic)242 try Data(markdown(report).utf8).write(to: reportDirectory.appendingPathComponent("mixed-batch.md"), options: .atomic)243 let failed = checks.filter { !$0.passed }.count244 io.out("Mixed-batch robustness run: \(checks.count) checks, \(failed) failed — \(Shared.seconds(report.seconds)). Report: \(reportDirectory.appendingPathComponent("mixed-batch.md").path)")245 return failed == 0 ? .success : .conversionFailed246 }247248 static func markdown(_ report: Report) -> String {249 func cell(_ text: String) -> String { text.replacingOccurrences(of: "|", with: "\\|").replacingOccurrences(of: "\n", with: " ") }250 var out = "# ORIX Convert — mixed-batch robustness run\n\n> Generated file — do not edit. Written by `orix-convert dev stress`.\n\n"251 out += "Everything goes through the `convert` command (queue, validation, safe writer); the harness only prepares the files and compares the\n"252 out += "outcomes with what the registry predicts for each file.\n\n"253 out += "| | |\n|---|---|\n| Date | \(report.date) |\n| Machine | \(report.machine) |\n| Load average (1, 5, 15 min) | \(report.loadAverage.map { String($0) }.joined(separator: ", ")) |\n"254 out += "| Files | \(report.files.values.reduce(0, +)): \(report.files.sorted { $0.key < $1.key }.map { "\($0.value) \($0.key)" }.joined(separator: ", ")) |\n"255 out += "| Duration | \(Shared.seconds(report.seconds)) |\n\n"256 out += "`valid` = generated PNG, WAV, TXT, Markdown, CSV, JSON, HTML and SVG files; `garbage` = random bytes under ten different extensions; `empty` = zero-byte files (a zero-byte `.txt` is a valid empty document and is expected to convert; the others are expected to fail);\n"257 out += "`collision` = valid files in sub-folders whose names repeat names of the batch; `deleted` = a valid file removed while its job was queued.\n\n"258 out += "## Scenarios\n\n| Scenario | Exit | Jobs | Finished | Failed | Skipped | Cancelled | Failures by kind |\n|---|---:|---:|---:|---:|---:|---:|---|\n"259 for s in report.scenarios {260 out += "| \(s.name) | \(s.exitCode) | \(s.summary.total) | \(s.summary.finished) | \(s.summary.failed) | \(s.summary.skipped) | \(s.summary.cancelled) | \(s.failuresByKind.isEmpty ? "—" : s.failuresByKind.sorted { $0.key < $1.key }.map { "\($0.key) \($0.value)" }.joined(separator: ", ")) |\n"261 }262 let failed = report.checks.filter { !$0.passed }.count263 out += "\n## Checks (\(report.checks.count), \(failed) failed)\n\n| Check | Result | Detail |\n|---|---|---|\n"264 for c in report.checks { out += "| \(cell(c.name)) | \(c.passed ? "passed" : "FAILED") | \(cell(c.detail)) |\n" }265 out += "\n## Outcomes that differ from the prediction (\(report.unexpected.count))\n\n"266 out += report.unexpected.isEmpty ? "None.\n" : report.unexpected.map { "- \($0)" }.joined(separator: "\n") + "\n"267 out += "\n## Temporary root after the run\n\n`\(report.temporaryRoot)` — \(report.temporaryRootEntries.count) entr\(report.temporaryRootEntries.count == 1 ? "y" : "ies")"268 out += report.temporaryRootEntries.isEmpty ? " (empty).\n" : " (work folders are named `<pid>-<uuid>`; any listed here belong to other running ORIX Convert processes on this shared Mac): \(report.temporaryRootEntries.prefix(20).joined(separator: ", ")).\n"269 return out270 }271}272