SPB Git

Repository Public License

Stress.swift 22.6 KB 272 lines swift

1// ORIX Convert — Packages/ConvertKit/Sources/ConvertCLI/Stress.swift2// Description: Mixed-batch robustness run (`orix-convert dev stress`): 300 files of mixed generated formats, garbage,3//              empty files, a file deleted mid-run, a read-only destination and name collisions, all through the4//              real `convert` command; checks every outcome against what the registry says, and that nothing leaks.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import UniformTypeIdentifiers8import ConvertCore9import ConvertKit1011enum Stress {12    struct Check: Codable { let name: String; let passed: Bool; let detail: String }13    struct Scenario: Codable {14        let name: String15        let command: String16        let exitCode: Int3217        let summary: BatchSummary18        let failuresByKind: [String: Int]19    }20    struct Report: Codable {21        let date: String, machine: String22        let loadAverage: [Double]23        let files: [String: Int]24        let scenarios: [Scenario]25        let checks: [Check]26        let unexpected: [String]27        let temporaryRoot: String28        let temporaryRootEntries: [String]29        let seconds: Double30    }3132    final class Buffer: @unchecked Sendable {33        private let lock = NSLock()34        private var lines: [String] = []35        func add(_ line: String) { lock.lock(); lines.append(line); lock.unlock() }36        var text: String { lock.lock(); defer { lock.unlock() }; return lines.joined(separator: "\n") }37    }3839    static func run(_ arguments: ParsedArguments, io: CommandIO) async throws -> ExitCode {40        let started = Date()41        let fm = FileManager.default42        let work = arguments.value("work-dir").map(Shared.fileURL) ?? fm.temporaryDirectory.appendingPathComponent("orix-convert-stress-\(ProcessInfo.processInfo.processIdentifier)", isDirectory: true)43        guard let reportDirectory = arguments.value("report-dir").map(Shared.fileURL) ?? Repository.root()?.appendingPathComponent("docs/reports") else {44            throw UsageError("dev stress needs --report-dir when it does not run from the repository.")45        }46        let jobs = try arguments.int("jobs", in: 1...32) ?? 647        try? fm.removeItem(at: work)48        let input = work.appendingPathComponent("in", isDirectory: true)49        try fm.createDirectory(at: input, withIntermediateDirectories: true)50        defer { if !arguments.flag("keep") { _ = chmod(work.appendingPathComponent("read-only").path, 0o755); try? fm.removeItem(at: work) } }5152        // 1. Material: 300 files.53        enum Kind: String { case valid, garbage, empty, collision, deleted }54        var files: [(url: URL, kind: Kind)] = []55        func add(_ name: String, _ data: Data, _ kind: Kind = .valid, folder: String? = nil) throws {56            let directory = folder.map { input.appendingPathComponent($0, isDirectory: true) } ?? input57            try fm.createDirectory(at: directory, withIntermediateDirectories: true)58            let url = directory.appendingPathComponent(name)59            try data.write(to: url)60            files.append((url, kind))61        }62        func png(_ seed: Int) throws -> Data {63            let url = work.appendingPathComponent("scratch.png")64            try Seeds.writeImage(try Seeds.image(width: 160 + seed % 5 * 32, height: 120 + seed % 3 * 24, seed: seed), to: url)65            return try Data(contentsOf: url)66        }67        func wav() throws -> Data {68            let url = work.appendingPathComponent("scratch.wav")69            try Seeds.writeWAV(to: url, seconds: 0.3)70            return try Data(contentsOf: url)71        }72        let wavData = try wav()73        for i in 1...60 { try add(String(format: "photo-%03d.png", i), try png(i)) }74        for i in 1...30 { try add(String(format: "sound-%03d.wav", i), wavData) }75        for i in 1...30 { try add(String(format: "note-%03d.txt", i), Data((Seeds.text + "\nNote \(i).\n").utf8)) }76        for i in 1...30 { try add(String(format: "page-%03d.md", i), Data((Seeds.markdown + "\n\nPage \(i).\n").utf8)) }77        for i in 1...30 { try add(String(format: "table-%03d.csv", i), Data(Seeds.csv(rows: 5 + i).utf8)) }78        for i in 1...30 { try add(String(format: "records-%03d.json", i), Data(Seeds.json(rows: 3 + i).utf8)) }79        for i in 1...20 { try add(String(format: "web-%03d.html", i), Data(Seeds.html.utf8)) }80        for i in 1...20 { try add(String(format: "drawing-%03d.svg", i), Data(Seeds.svg.utf8)) }81        var state: UInt64 = 4282        func noise(_ count: Int) -> Data {83            var bytes = [UInt8](repeating: 0, count: count)84            for i in bytes.indices { state = state &* 6364136223846793005 &+ 1442695040888963407; bytes[i] = UInt8(truncatingIfNeeded: state >> 56) }85            bytes[0] = 0; bytes[1] = 0xFF; bytes[2] = 086            return Data(bytes)87        }88        let garbageExtensions = ["png", "pdf", "docx", "mp3", "zip", "jpg", "xlsx", "mp4", "bin", "dat"]89        for i in 1...20 { try add(String(format: "garbage-%03d.%@", i, garbageExtensions[i % garbageExtensions.count]), noise(2000 + i * 37), .garbage) }90        let emptyExtensions = ["png", "txt", "pdf", "csv", "json"]91        for i in 1...15 { try add(String(format: "empty-%03d.%@", i, emptyExtensions[i % emptyExtensions.count]), Data(), .empty) }92        for i in 1...14 { try add(String(format: "photo-%03d.png", i % 7 + 1), try png(100 + i), .collision, folder: "dup-\(i)") }93        try add("vanishing.png", try png(999), .deleted)94        try? fm.removeItem(at: work.appendingPathComponent("scratch.png"))95        try? fm.removeItem(at: work.appendingPathComponent("scratch.wav"))96        let counts = Dictionary(grouping: files, by: { $0.kind.rawValue }).mapValues(\.count)97        io.err("Mixed batch: \(files.count) files (\(counts.sorted { $0.key < $1.key }.map { "\($0.value) \($0.key)" }.joined(separator: ", "))).")9899        var checks: [Check] = []100        var unexpected: [String] = []101        var scenarios: [Scenario] = []102        func check(_ name: String, _ passed: Bool, _ detail: String) {103            checks.append(Check(name: name, passed: passed, detail: detail))104            io.err("  \(passed ? "ok  " : "FAIL") \(name): \(detail)")105        }106        func convert(_ name: String, _ argv: [String], afterEnqueue: (@Sendable (ConversionService, [UUID]) async -> Void)? = nil) async throws -> BatchReport {107            let out = Buffer(), err = Buffer()108            let captured = CommandIO(out: out.add, err: err.add, environment: io.environment)109            let parsed = try ArgumentParser.parse(argv, spec: ConvertCommand.spec)110            let (code, report) = try await ConvertCommand.run(parsed, io: captured, configure: nil, afterEnqueue: afterEnqueue)111            // The JSON the command printed must decode to the same summary: that is what a script would read.112            if let data = out.text.data(using: .utf8), let decoded = try? JSONDecoder().decode(BatchReport.self, from: data) {113                check("\(name): JSON summary matches the jobs", decoded.summary.total == decoded.jobs.count114                      && decoded.summary.finished == decoded.jobs.filter { $0.state == "finished" }.count115                      && decoded.summary.failed == decoded.jobs.filter { $0.state == "failed" }.count116                      && decoded.summary.skipped == decoded.jobs.filter { $0.state == "skipped" }.count117                      && decoded.summary.finished + decoded.summary.failed + decoded.summary.skipped + decoded.summary.cancelled == decoded.summary.total,118                      "\(decoded.summary.total) jobs: \(decoded.summary.finished) finished, \(decoded.summary.failed) failed, \(decoded.summary.skipped) skipped, \(decoded.summary.cancelled) cancelled")119            } else {120                check("\(name): JSON output decodes", false, "the command did not print a decodable report")121            }122            let kinds = Dictionary(grouping: report.jobs.filter { $0.state == "failed" }, by: { $0.errorKind ?? "?" }).mapValues(\.count)123            scenarios.append(Scenario(name: name, command: "orix-convert convert " + argv.map { $0.hasPrefix(work.path) ? "<work>" + $0.dropFirst(work.path.count) : $0 }.suffix(12).joined(separator: " "),124                                      exitCode: code.rawValue, summary: report.summary, failuresByKind: kinds))125            check("\(name): no work folder left", report.leftoverTemporaryItems.isEmpty, report.leftoverTemporaryItems.isEmpty ? "none" : report.leftoverTemporaryItems.joined(separator: ", "))126            return report127        }128        func partials(in directory: URL) -> [String] {129            guard let enumerator = fm.enumerator(at: directory, includingPropertiesForKeys: nil, options: []) else { return [] }130            var found: [String] = []131            for case let url as URL in enumerator where url.lastPathComponent.contains(".partial") || url.lastPathComponent.hasPrefix(".orix-convert") { found.append(url.lastPathComponent) }132            return found133        }134135        // 2. Scenario A: everything to PDF, one output folder, names colliding, one file deleted while queued.136        let registry = Shared.registry()137        let planner = ConversionPlanner(registry: registry)138        var expected: [String: String] = [:]139        for file in files {140            switch file.kind {141            case .garbage, .deleted: expected[file.url.path] = "failed"142            case .empty:143                // A zero-byte .txt is a valid, empty text document (it becomes a blank page); every other empty file is unreadable.144                expected[file.url.path] = file.url.pathExtension == "txt" ? "finished" : "failed"145            case .valid, .collision:146                let format = FileTypeDetector.detect(file.url).format147                expected[file.url.path] = format.flatMap { try? planner.plan(from: $0, to: .pdf) } == nil ? "failed" : "finished"148            }149        }150        let outA = work.appendingPathComponent("out-a", isDirectory: true)151        let vanishing = files.last { $0.kind == .deleted }?.url152        let reportA = try await convert("mixed batch to PDF", files.map(\.url.path) + ["--to", "pdf", "--output-dir", outA.path, "--jobs", String(jobs), "--json", "--no-history"]) { _, _ in153            if let vanishing { try? FileManager.default.removeItem(at: vanishing) }154        }155        var mismatches = 0156        for job in reportA.jobs {157            guard let source = job.sources.first else { continue }158            let want = expected[source] ?? "?"159            if job.state != want {160                mismatches += 1161                unexpected.append("\((source as NSString).lastPathComponent): expected \(want), got \(job.state)\(job.error.map { " — \($0)" } ?? "")")162            }163        }164        check("mixed batch: every outcome is the one the registry predicts", mismatches == 0, "\(reportA.jobs.count) jobs, \(mismatches) unexpected outcome(s)")165        check("mixed batch: one job per file", reportA.jobs.count == files.count, "\(reportA.jobs.count) jobs for \(files.count) files")166        let expectedFinished = expected.values.filter { $0 == "finished" }.count167        check("mixed batch: summary counts", reportA.summary.finished == expectedFinished && reportA.summary.failed == files.count - expectedFinished && reportA.summary.skipped == 0,168              "finished \(reportA.summary.finished) (expected \(expectedFinished)), failed \(reportA.summary.failed) (expected \(files.count - expectedFinished))")169        check("mixed batch: exit status 1 (some conversions failed)", reportA.summary.exitCode == ExitCode.conversionFailed.rawValue, "exit \(reportA.summary.exitCode)")170        let writtenA = reportA.jobs.flatMap(\.outputs).map(\.path)171        check("mixed batch: every reported output exists and no two jobs share one", writtenA.allSatisfy { fm.fileExists(atPath: $0) } && Set(writtenA).count == writtenA.count,172              "\(writtenA.count) outputs, \(Set(writtenA).count) distinct")173        let renamed = reportA.jobs.flatMap(\.outputs).filter { $0.outcome == "renamed" }.count174        let collidingValid = files.filter { $0.kind == .collision }.count175        check("mixed batch: name collisions were numbered, not overwritten", renamed == collidingValid, "\(renamed) outputs renamed for \(collidingValid) colliding names")176        let garbageJobs = reportA.jobs.filter { ($0.sources.first.map { ($0 as NSString).lastPathComponent } ?? "").hasPrefix("garbage-") || ($0.sources.first.map { ($0 as NSString).lastPathComponent } ?? "").hasPrefix("empty-") }177        let emptyText = garbageJobs.filter { ($0.sources.first ?? "").hasSuffix(".txt") }178        check("mixed batch: an empty .txt is converted as an empty document", emptyText.allSatisfy { $0.state == "finished" }, "\(emptyText.count) empty .txt file(s): \(emptyText.filter { $0.state == "finished" }.count) converted to a blank PDF")179        check("mixed batch: garbage and the other empty files fail with a message and write nothing",180              garbageJobs.filter { !($0.sources.first ?? "").hasSuffix(".txt") }.allSatisfy { $0.state == "failed" && !($0.error ?? "").isEmpty && $0.outputs.isEmpty },181              "\(garbageJobs.count) files; kinds: \(Dictionary(grouping: garbageJobs, by: { $0.errorKind ?? "?" }).mapValues(\.count).sorted { $0.key < $1.key }.map { "\($0.key) \($0.value)" }.joined(separator: ", "))")182        if let gone = reportA.jobs.first(where: { $0.sources.first == vanishing?.path }) {183            check("mixed batch: the file deleted while queued fails cleanly", gone.state == "failed" && gone.errorKind == "unreadableInput", gone.error ?? gone.state)184        }185        check("mixed batch: no partial file in the output folder", partials(in: outA).isEmpty, partials(in: outA).isEmpty ? "none" : partials(in: outA).joined(separator: ", "))186        let sourcesIntact = files.filter { $0.kind != .deleted }.allSatisfy { fm.fileExists(atPath: $0.url.path) }187        check("mixed batch: no source file was touched", sourcesIntact, sourcesIntact ? "all \(files.count - 1) remaining sources present" : "a source is missing")188189        // 3. Scenario B: read-only destination.190        let readOnly = work.appendingPathComponent("read-only", isDirectory: true)191        try fm.createDirectory(at: readOnly, withIntermediateDirectories: true)192        _ = chmod(readOnly.path, 0o555)193        let ten = files.filter { $0.kind == .valid && $0.url.pathExtension == "png" }.prefix(10).map(\.url.path)194        let reportB = try await convert("read-only destination", ten + ["--to", "jpeg", "--output-dir", readOnly.path, "--jobs", String(jobs), "--json", "--no-history"])195        check("read-only destination: every job fails with a write error", reportB.summary.failed == 10 && reportB.jobs.allSatisfy { $0.errorKind == "writeFailed" },196              "failed \(reportB.summary.failed)/10; kinds: \(Set(reportB.jobs.compactMap(\.errorKind)).sorted().joined(separator: ", "))")197        _ = chmod(readOnly.path, 0o755)198        let leftB = (try? fm.contentsOfDirectory(atPath: readOnly.path)) ?? []199        check("read-only destination: nothing was written there", leftB.isEmpty, leftB.isEmpty ? "empty" : leftB.joined(separator: ", "))200201        // 4. Scenario C: an existing destination under each conflict policy.202        let outC = work.appendingPathComponent("out-c", isDirectory: true)203        let first = try await convert("conflicts: first write", ten + ["--to", "jpeg", "--output-dir", outC.path, "--json", "--no-history"])204        check("conflicts: first write succeeds", first.summary.finished == 10, "finished \(first.summary.finished)/10")205        func snapshot() -> [String: Data] {206            var result: [String: Data] = [:]207            for name in (try? fm.contentsOfDirectory(atPath: outC.path)) ?? [] { result[name] = try? Data(contentsOf: outC.appendingPathComponent(name)) }208            return result209        }210        let original = snapshot()211        let failRun = try await convert("conflicts: --on-conflict fail", ten + ["--to", "jpeg", "--output-dir", outC.path, "--on-conflict", "fail", "--json", "--no-history"])212        check("conflicts: fail → exit 5, nothing written or changed", failRun.summary.exitCode == ExitCode.destinationExists.rawValue && failRun.summary.failed == 10 && snapshot() == original,213              "exit \(failRun.summary.exitCode), failed \(failRun.summary.failed), folder unchanged: \(snapshot() == original)")214        let skipRun = try await convert("conflicts: --on-conflict skip", ten + ["--to", "jpeg", "--output-dir", outC.path, "--on-conflict", "skip", "--json", "--no-history"])215        check("conflicts: skip → exit 0, 10 skipped, nothing changed", skipRun.summary.exitCode == 0 && skipRun.summary.skipped == 10 && snapshot() == original,216              "exit \(skipRun.summary.exitCode), skipped \(skipRun.summary.skipped), folder unchanged: \(snapshot() == original)")217        let renameRun = try await convert("conflicts: default (rename)", ten + ["--to", "jpeg", "--output-dir", outC.path, "--json", "--no-history"])218        let afterRename = snapshot()219        check("conflicts: rename → 10 new numbered files, originals intact", renameRun.summary.finished == 10 && afterRename.count == 20 && original.allSatisfy { afterRename[$0.key] == $0.value },220              "finished \(renameRun.summary.finished), \(afterRename.count) files in the folder")221        let replaceRun = try await convert("conflicts: --on-conflict replace", ten + ["--to", "jpeg", "--option", "quality=30", "--output-dir", outC.path, "--on-conflict", "replace", "--json", "--no-history"])222        let afterReplace = snapshot()223        let replaced = original.keys.filter { afterReplace[$0] != original[$0] }.count224        check("conflicts: replace → the 10 files are replaced, no extra file", replaceRun.summary.finished == 10 && afterReplace.count == 20 && replaced == 10,225              "finished \(replaceRun.summary.finished), replaced \(replaced), \(afterReplace.count) files")226        check("conflicts: no partial file left", partials(in: outC).isEmpty, partials(in: outC).isEmpty ? "none" : partials(in: outC).joined(separator: ", "))227        let sourceReplace = try await convert("source protection", [ten[0], "--to", "png", "--on-conflict", "replace", "--json", "--no-history"])228        check("source protection: converting a file onto itself with replace is refused", sourceReplace.summary.failed == 1 && sourceReplace.jobs.first?.errorKind == "destinationExists" && fm.fileExists(atPath: ten[0]),229              sourceReplace.jobs.first?.error ?? "state \(sourceReplace.jobs.first?.state ?? "?")")230231        // 5. The application's temporary root.232        let root = AppPaths.temporaryRoot233        let entries = ((try? fm.contentsOfDirectory(atPath: root.path)) ?? []).sorted()234        let ours = entries.filter { $0.hasPrefix("\(ProcessInfo.processInfo.processIdentifier)-") }235        check("temporary root holds nothing of this run", ours.isEmpty, "\(root.path): \(entries.count) entr\(entries.count == 1 ? "y" : "ies") in total, \(ours.count) from this process")236237        let report = Report(date: ISO8601DateFormatter().string(from: started), machine: Sweep.machine(), loadAverage: Sweep.loadAverage(), files: counts,238                            scenarios: scenarios, checks: checks, unexpected: unexpected, temporaryRoot: root.path, temporaryRootEntries: entries,239                            seconds: Date().timeIntervalSince(started))240        try fm.createDirectory(at: reportDirectory, withIntermediateDirectories: true)241        try Data((try Shared.json(report) + "\n").utf8).write(to: reportDirectory.appendingPathComponent("mixed-batch.json"), options: .atomic)242        try Data(markdown(report).utf8).write(to: reportDirectory.appendingPathComponent("mixed-batch.md"), options: .atomic)243        let failed = checks.filter { !$0.passed }.count244        io.out("Mixed-batch robustness run: \(checks.count) checks, \(failed) failed — \(Shared.seconds(report.seconds)). Report: \(reportDirectory.appendingPathComponent("mixed-batch.md").path)")245        return failed == 0 ? .success : .conversionFailed246    }247248    static func markdown(_ report: Report) -> String {249        func cell(_ text: String) -> String { text.replacingOccurrences(of: "|", with: "\\|").replacingOccurrences(of: "\n", with: " ") }250        var out = "# ORIX Convert — mixed-batch robustness run\n\n> Generated file — do not edit. Written by `orix-convert dev stress`.\n\n"251        out += "Everything goes through the `convert` command (queue, validation, safe writer); the harness only prepares the files and compares the\n"252        out += "outcomes with what the registry predicts for each file.\n\n"253        out += "| | |\n|---|---|\n| Date | \(report.date) |\n| Machine | \(report.machine) |\n| Load average (1, 5, 15 min) | \(report.loadAverage.map { String($0) }.joined(separator: ", ")) |\n"254        out += "| Files | \(report.files.values.reduce(0, +)): \(report.files.sorted { $0.key < $1.key }.map { "\($0.value) \($0.key)" }.joined(separator: ", ")) |\n"255        out += "| Duration | \(Shared.seconds(report.seconds)) |\n\n"256        out += "`valid` = generated PNG, WAV, TXT, Markdown, CSV, JSON, HTML and SVG files; `garbage` = random bytes under ten different extensions; `empty` = zero-byte files (a zero-byte `.txt` is a valid empty document and is expected to convert; the others are expected to fail);\n"257        out += "`collision` = valid files in sub-folders whose names repeat names of the batch; `deleted` = a valid file removed while its job was queued.\n\n"258        out += "## Scenarios\n\n| Scenario | Exit | Jobs | Finished | Failed | Skipped | Cancelled | Failures by kind |\n|---|---:|---:|---:|---:|---:|---:|---|\n"259        for s in report.scenarios {260            out += "| \(s.name) | \(s.exitCode) | \(s.summary.total) | \(s.summary.finished) | \(s.summary.failed) | \(s.summary.skipped) | \(s.summary.cancelled) | \(s.failuresByKind.isEmpty ? "—" : s.failuresByKind.sorted { $0.key < $1.key }.map { "\($0.key) \($0.value)" }.joined(separator: ", ")) |\n"261        }262        let failed = report.checks.filter { !$0.passed }.count263        out += "\n## Checks (\(report.checks.count), \(failed) failed)\n\n| Check | Result | Detail |\n|---|---|---|\n"264        for c in report.checks { out += "| \(cell(c.name)) | \(c.passed ? "passed" : "FAILED") | \(cell(c.detail)) |\n" }265        out += "\n## Outcomes that differ from the prediction (\(report.unexpected.count))\n\n"266        out += report.unexpected.isEmpty ? "None.\n" : report.unexpected.map { "- \($0)" }.joined(separator: "\n") + "\n"267        out += "\n## Temporary root after the run\n\n`\(report.temporaryRoot)` — \(report.temporaryRootEntries.count) entr\(report.temporaryRootEntries.count == 1 ? "y" : "ies")"268        out += report.temporaryRootEntries.isEmpty ? " (empty).\n" : " (work folders are named `<pid>-<uuid>`; any listed here belong to other running ORIX Convert processes on this shared Mac): \(report.temporaryRootEntries.prefix(20).joined(separator: ", ")).\n"269        return out270    }271}272
Files main
Menu

Theme