Repository Public License
1// ORIX Convert — Packages/ConvertKit/Sources/ConvertCLI/Sweep.swift2// Description: The cross-engine end-to-end sweep (`orix-convert matrix --verify`). Every declared path is run exactly3// as a user would run it — detector, planner, queue, engine, generic and engine validation, safe writer —4// on a real fixture, with bounded concurrency, a per-path time limit and resumable state.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import ConvertCore8import ConvertKit910struct SweepResult: Codable, Sendable {11 var id: String12 var kind: String // "path" or "chain"13 var source: String, destination: String14 var operation: String?15 var engines: [String]16 var status: String // pass, fail, skip, timeout, cancelled, crash; chains only: refused (content)17 var seconds: Double?18 var error: String?19 var errorKind: String?20 var failedCheck: String?21 /// What the detector says the fixture is, when that differs from the path's source format.22 var detectedAs: String?23 /// The job was given the source format explicitly because the detector names another format for the fixture.24 var explicitSourceFormat: Bool25 var fixtureOrigin: String?26 /// 0 = the primary fixture of the source format; above, an alternative real file of the same format.27 var fixtureVariant: Int?28 /// When the path passed on a variant only: what happened on the primary fixture.29 var primaryFixtureError: String?30 var primaryFixtureOrigin: String?31 var primaryErrorKind: String?32 /// For a failure: how many different fixtures were tried.33 var variantsTried: Int?34 /// Set when an earlier invocation of the sweep crashed while this path was running: the paths that were running35 /// with it. The result recorded here is the one of the re-run, alone, in a child process.36 var crashedWith: [String]?37 var options: [String: String]38 var checks: Int39 var warnings: [String]40 var outputs: Int41 var outputBytes: Int64?42 var note: String?43 /// For a chain reported as "refused": which content-dependent refusal the engine explained (`ContentRefusal`).44 var refusalReason: String?45}4647struct SweepState: Codable {48 var fingerprint: String49 var results: [String: SweepResult]50}5152struct SweepReport: Codable {53 struct Environment: Codable {54 let date: String, tool: String, machine: String, system: String55 let loadAverageStart: [Double], loadAverageEnd: [Double]56 let jobs: Int, heavyJobs: Int, timeoutSeconds: Double57 let path: String58 let toolsEnvironment: String?59 let wallSeconds: Double60 let filter: String?61 }62 struct Totals: Codable {63 var total = 0, pass = 0, passOnVariantOnly = 0, fail = 0, crash = 0, skip = 0, timeout = 0, cancelled = 064 /// Chained plans only: the engine refused the fixture for a reason that depends on its content.65 var refusedContent = 066 }67 struct DetectorNote: Codable { let format: String; let detectedAs: String?; let evidence: String; let confidence: String; let fixtureOrigin: String; let acceptedAsEquivalent: Bool }68 struct InspectionNote: Codable { let format: String; let engine: String?; let error: String?; let engineFields: Int }69 struct ChainPair: Codable { let source: String; let destination: String; let through: String; let engines: [String]; let fidelity: String }70 struct Chains: Codable {71 let totalPairs: Int72 let byEngines: [String: Int]73 let sampled: Int74 let totals: Totals75 let pairs: [ChainPair]76 }77 let environment: Environment78 let registryFingerprint: String79 let totals: Totals80 let byEngine: [String: Totals]81 let fixtures: [Fixture]82 let formatsWithoutFixture: [String: String]83 let detectorNotes: [DetectorNote]84 let inspections: [InspectionNote]85 let chains: Chains?86 let temporaryLeftovers: [String]87 let results: [SweepResult]88 let chainResults: [SweepResult]89}9091enum FormatTypeCheck {92 /// The folder seed is a directory: there is nothing to inspect in it.93 static func isFolderFixture(_ url: URL, format: FormatID) -> Bool { format == .folder }94}9596enum Sweep {97 /// Options a named operation needs to do anything at all (its defaults ask for "nothing"). Everything else runs98 /// with the path's defaults, as a user who changes no setting.99 static let operationOptions: [String: [String: OptionValue]] = [100 "pdf.extract-pages": ["pages": .string("1-2")],101 "pdf.delete-pages": ["pages": .string("1")],102 "pdf.set-metadata": ["metadata.title": .string("ORIX Convert sweep")],103 ]104105 static func loadAverage() -> [Double] {106 var values = [Double](repeating: 0, count: 3)107 getloadavg(&values, 3)108 return values.map { ($0 * 100).rounded() / 100 }109 }110111 static func machine() -> String {112 func sysctl(_ name: String) -> String {113 var size = 0114 sysctlbyname(name, nil, &size, nil, 0)115 var buffer = [CChar](repeating: 0, count: max(1, size))116 sysctlbyname(name, &buffer, &size, nil, 0)117 return String(cString: buffer)118 }119 let memory = ProcessInfo.processInfo.physicalMemory / (1 << 30)120 return "\(sysctl("machdep.cpu.brand_string")), \(ProcessInfo.processInfo.processorCount) cores, \(memory) GB (\(sysctl("hw.model")))"121 }122123 final class Collector: @unchecked Sendable {124 private let lock = NSLock()125 private var state: SweepState126 private let url: URL127 private var unsaved = 0128 var timedOut = Set<UUID>()129130 init(state: SweepState, url: URL) { self.state = state; self.url = url }131132 func record(_ result: SweepResult) -> (done: Int, pass: Int, fail: Int, skip: Int) {133 lock.lock(); defer { lock.unlock() }134 state.results[result.id] = result135 unsaved += 1136 if unsaved >= 20 { saveLocked() }137 let values = state.results.values138 return (values.count, values.filter { $0.status == "pass" }.count, values.filter { $0.status == "fail" || $0.status == "timeout" || $0.status == "crash" }.count,139 values.filter { $0.status == "skip" }.count)140 }141142 private var jobPaths: [UUID: String] = [:]143 private var lastInFlight: [String] = []144 var inFlightURL: URL { url.deletingLastPathComponent().appendingPathComponent("in-flight.json") }145146 func bind(_ job: UUID, to path: String) { lock.lock(); jobPaths[job] = path; lock.unlock() }147148 /// Journal of the paths being converted right now. If the process dies (an engine crash takes the whole149 /// process with it), the next invocation finds the suspects here.150 func noteActive(_ jobs: [UUID]) {151 lock.lock()152 let paths = jobs.compactMap { jobPaths[$0] }.sorted()153 let changed = paths != lastInFlight154 lastInFlight = paths155 lock.unlock()156 if changed, let data = try? JSONEncoder().encode(paths) { try? data.write(to: inFlightURL, options: .atomic) }157 }158159 func clearInFlight() { try? FileManager.default.removeItem(at: inFlightURL) }160161 func markTimedOut(_ id: UUID) { lock.lock(); timedOut.insert(id); lock.unlock() }162 func didTimeOut(_ id: UUID) -> Bool { lock.lock(); defer { lock.unlock() }; return timedOut.contains(id) }163 func result(_ id: String) -> SweepResult? { lock.lock(); defer { lock.unlock() }; return state.results[id] }164 var results: [String: SweepResult] { lock.lock(); defer { lock.unlock() }; return state.results }165 func save() { lock.lock(); saveLocked(); lock.unlock() }166167 private func saveLocked() {168 unsaved = 0169 if let text = try? Shared.json(state) { try? Data(text.utf8).write(to: url, options: .atomic) }170 }171 }172173 static func run(_ arguments: ParsedArguments, io: CommandIO, repository: URL?) async throws -> ExitCode {174 let started = Date()175 let loadStart = loadAverage()176 let jobs = try arguments.int("jobs", in: 1...32) ?? 6177 let heavy = try arguments.int("heavy-jobs", in: 1...32) ?? max(2, jobs / 2)178 let timeout = try arguments.double("timeout", in: 5...7200) ?? 300179 let chainSample = arguments.flag("no-chains") ? 0 : (try arguments.int("chains", in: 0...2000) ?? 40)180 let workDirectory = arguments.value("work-dir").map(Shared.fileURL) ?? FileManager.default.temporaryDirectory.appendingPathComponent("orix-convert-e2e", isDirectory: true)181 guard let reportDirectory = arguments.value("report-dir").map(Shared.fileURL) ?? repository?.appendingPathComponent("docs/reports") else {182 throw UsageError("matrix --verify needs --report-dir when it does not run from the repository.")183 }184 let fm = FileManager.default185 if arguments.flag("fresh") { try? fm.removeItem(at: workDirectory) }186 try fm.createDirectory(at: workDirectory, withIntermediateDirectories: true)187 try fm.createDirectory(at: reportDirectory, withIntermediateDirectories: true)188189 let service = ConversionService(configuration: .init(maxConcurrentJobs: jobs, maxHeavyJobs: heavy, recordHistory: false))190 let registry = service.registry191 let document = MatrixBuilder.build(registry: registry, repository: repository)192 let fingerprint = document.identity.registryFingerprint193 let engineFilter = arguments.value("engine")194 let from = try arguments.value("from").map { try Shared.format($0, option: "--from") }195 let to = try arguments.value("to").map { try Shared.format($0, option: "--to") }196 let paths = registry.paths(from: from, to: to, engine: engineFilter)197 let filterText = [engineFilter.map { "engine=\($0)" }, from.map { "from=\($0)" }, to.map { "to=\($0)" }].compactMap { $0 }.joined(separator: " ")198199 let stateURL = workDirectory.appendingPathComponent("state.json")200 var state = SweepState(fingerprint: fingerprint, results: [:])201 if let data = try? Data(contentsOf: stateURL), let saved = try? JSONDecoder().decode(SweepState.self, from: data) {202 if saved.fingerprint == fingerprint {203 state = saved204 } else {205 io.err("The registry changed since the saved state (fingerprint differs): results of paths that still exist are kept.")206 let ids = Set(registry.paths.map(\.id))207 state.results = saved.results.filter { ids.contains($0.key) || $0.value.kind == "chain" }208 }209 }210 if arguments.flag("retry-failed") { state.results = state.results.filter { $0.value.status == "pass" } }211 // Skips are always re-evaluated: a fixture may exist now.212 state.results = state.results.filter { $0.value.status != "skip" && $0.value.status != "cancelled" }213 let collector = Collector(state: state, url: stateURL)214215 // Fixtures.216 io.err("Preparing fixtures in \(workDirectory.path) …")217 let store = FixtureStore(directory: workDirectory.appendingPathComponent("fixtures", isDirectory: true), service: service, log: { io.err(" " + $0) })218 await store.loadManifest()219 let childMode = arguments.value("only-id") != nil220 if !childMode { await store.build(for: Set(registry.paths.map(\.source))) }221 let missing = await store.missing222 let primaryFormats = Set(await store.fixtures.keys)223 io.err("Fixtures: \(primaryFormats.count) format(s); \(missing.count) source format(s) without a fixture.")224225 // Time limit: counted from the start of each job.226 let queue = service.queue227 let watchdog = Task {228 while !Task.isCancelled {229 try? await Task.sleep(nanoseconds: 500_000_000)230 let now = Date()231 let active = await queue.snapshot().jobs.filter { $0.state.isActive }232 collector.noteActive(active.map(\.id))233 for job in active where !job.cancelRequested {234 if let start = job.startedAt, now.timeIntervalSince(start) > timeout {235 collector.markTimedOut(job.id)236 await queue.cancel(job.id)237 }238 }239 }240 }241 let interrupt = InterruptCenter.shared.register { Task { await queue.cancelAll() } }242 defer { InterruptCenter.shared.unregister(interrupt) }243244 let outRoot = workDirectory.appendingPathComponent("out", isDirectory: true)245 if !childMode { try? fm.removeItem(at: outRoot) }246 let keep = arguments.flag("keep-outputs")247 let screen = Screen(io: io, showsProgress: true)248 let totalPlanned = paths.count249250 @Sendable func execute(id: String, kind: String, source: FormatID, destination: FormatID, operation: String?, engines: [String], engineID: String?,251 multi: Bool, index: Int) async -> SweepResult {252 var result = SweepResult(id: id, kind: kind, source: source.rawValue, destination: destination.rawValue, operation: operation, engines: engines,253 status: "skip", seconds: nil, error: nil, errorKind: nil, failedCheck: nil, detectedAs: nil, explicitSourceFormat: false,254 fixtureOrigin: nil, fixtureVariant: nil, primaryFixtureError: nil, primaryFixtureOrigin: nil, primaryErrorKind: nil, variantsTried: nil, crashedWith: nil, options: [:], checks: 0, warnings: [], outputs: 0, outputBytes: nil, note: nil, refusalReason: nil)255 guard primaryFormats.contains(source) else {256 result.note = "No fixture of \(source.rawValue): \(missing[source] ?? "not produced")."257 return result258 }259 // The primary fixture first; when the path cannot work on it, other real files of the same format260 // (seed variants, the same format written by another path). A pass on a variant is reported as such,261 // with the primary's error.262 var first: SweepResult?263 var variant = 0264 while let fixture = await store.fixture(source, index: variant, hint: (destination, operation)) {265 var attempt = result266 attempt.fixtureOrigin = fixture.origin267 attempt.fixtureVariant = variant268 let detection = FileTypeDetector.detect(fixture.url)269 let explicit = detection.format != source270 if explicit { attempt.detectedAs = detection.format?.rawValue ?? "unrecognised"; attempt.explicitSourceFormat = true }271 var sources = [fixture.url]272 if multi {273 let second = fixture.url.deletingLastPathComponent().appendingPathComponent("fixture-b" + (fixture.url.pathExtension.isEmpty ? "" : "." + fixture.url.pathExtension))274 if !FileManager.default.fileExists(atPath: second.path) { try? FileManager.default.copyItem(at: fixture.url, to: second) }275 if FileManager.default.fileExists(atPath: second.path) { sources.append(second) }276 }277 let options = ConversionOptions(operation.flatMap { operationOptions[$0] } ?? [:])278 attempt.options = options.values.mapValues(Shared.optionValueText)279 let output = outRoot.appendingPathComponent("\(kind)-\(index)-\(variant)", isDirectory: true)280 let request = ConversionJobRequest(sources: sources, sourceFormat: explicit ? source : nil, destinationFormat: destination, operation: operation,281 options: options, engineID: engineID, destination: OutputDestination(directory: output, conflictPolicy: .rename),282 allowChaining: kind == "chain", recordInHistory: false)283 let jobID = await queue.enqueue(request)284 collector.bind(jobID, to: id)285 let job = await queue.waitForJob(jobID)286 if !keep { try? FileManager.default.removeItem(at: output) }287 guard let job else { attempt.status = "fail"; attempt.error = "The job disappeared from the queue."; return attempt }288 attempt.seconds = job.duration289 attempt.checks = job.checks.count290 attempt.warnings = job.warnings.map(\.code)291 attempt.outputs = job.outputs.count292 attempt.outputBytes = job.bytesAfter293 if kind == "chain", let plan = job.plan { attempt.engines = plan.engines; attempt.note = ([plan.source] + plan.steps.map(\.destination)).map(\.rawValue).joined(separator: " → ") }294 switch job.state {295 case .finished:296 attempt.status = "pass"297 if job.outputs.isEmpty { attempt.status = "fail"; attempt.error = "The job finished without any output."; attempt.errorKind = "outputInvalid" }298 case .cancelled:299 if collector.didTimeOut(jobID) {300 attempt.status = "timeout"; attempt.error = "Not finished \(Int(timeout)) s after it started; cancelled."301 } else {302 attempt.status = "cancelled"303 }304 default:305 attempt.status = "fail"306 attempt.error = job.error ?? "Ended in state \(job.state.rawValue)."307 attempt.errorKind = job.failure?.kind308 attempt.failedCheck = job.checks.first(where: { !$0.passed })?.name309 }310 _ = await queue.remove(jobID)311 if attempt.status == "pass" {312 if let first { attempt.primaryFixtureError = first.error; attempt.primaryFixtureOrigin = first.fixtureOrigin; attempt.primaryErrorKind = first.errorKind }313 return attempt314 }315 if first == nil { first = attempt }316 if attempt.status != "fail" { break }317 variant += 1318 }319 if var first {320 first.variantsTried = variant + 1321 // A chain hands the second step whatever the first one wrote. When no fixture can satisfy it and the322 // engine says why in terms of the content, that is a refusal of the content, not a failure of the plan.323 if kind == "chain", first.status == "fail", let reason = ContentRefusal.reason(kind: first.errorKind, message: first.error) {324 first.status = "refused"325 first.refusalReason = reason326 }327 return first328 }329 return result330 }331332 // Child mode: one path, alone in this process; the result goes to standard output as JSON.333 if let only = arguments.value("only-id") {334 guard let path = registry.paths.first(where: { $0.id == only }) else { throw UsageError("--only-id: no path '\(only)'.") }335 let result = await execute(id: path.id, kind: "path", source: path.source, destination: path.destination, operation: path.operation, engines: [path.engine],336 engineID: path.engine, multi: path.acceptsMultipleSources, index: Int(ProcessInfo.processInfo.processIdentifier))337 watchdog.cancel()338 io.out(try Shared.json(result))339 return .success340 }341342 // Paths that were running when an earlier invocation died are not run with the others again: each is343 // re-run alone in a child process, so a crash is attributed to one path and cannot take the sweep down.344 var suspects: [String] = []345 if let data = try? Data(contentsOf: collector.inFlightURL), let list = try? JSONDecoder().decode([String].self, from: data) {346 suspects = list.filter { collector.result($0) == nil }347 if !suspects.isEmpty { io.err("The previous invocation ended while \(suspects.count) path(s) were running: \(suspects.joined(separator: ", ")). They are re-run alone, in child processes.") }348 }349 collector.clearInFlight()350 for suspect in suspects {351 guard let path = paths.first(where: { $0.id == suspect }) else { continue }352 var result = await runInChild(path: path, workDirectory: workDirectory, timeout: timeout)353 result.crashedWith = suspects.filter { $0 != suspect }354 _ = collector.record(result)355 screen.error("re-run alone: \(result.status.uppercased()) \(result.id)\(result.error.map { ": \($0)" } ?? "")")356 }357 collector.save()358359 let pending = paths.enumerated().filter { collector.result($0.element.id) == nil }360 io.err("Running \(pending.count) of \(totalPlanned) path(s) (\(totalPlanned - pending.count) already recorded) — \(jobs) at once, \(heavy) heavy, limit \(Int(timeout)) s each.")361 await withTaskGroup(of: SweepResult.self) { group in362 for (index, path) in pending {363 group.addTask {364 await execute(id: path.id, kind: "path", source: path.source, destination: path.destination, operation: path.operation, engines: [path.engine],365 engineID: path.engine, multi: path.acceptsMultipleSources, index: index)366 }367 }368 for await result in group {369 let counts = collector.record(result)370 if result.status == "fail" || result.status == "timeout" {371 screen.error("\(result.status.uppercased()) \(result.id): \(result.error ?? "")")372 }373 screen.progress("[\(counts.done)] pass \(counts.pass) · fail \(counts.fail) · skip \(counts.skip) — \(result.id)")374 }375 }376 collector.save()377 collector.clearInFlight()378 screen.clear()379380 // Chained plans: every pair the planner bridges, and a sample actually run.381 var chainReport: SweepReport.Chains?382 var chainResults: [SweepResult] = []383 if engineFilter == nil, from == nil, to == nil, !InterruptCenter.shared.isInterrupted {384 let resolver = service.resolver, planner = service.planner385 var pairs: [(pair: SweepReport.ChainPair, plan: ConversionPlan)] = []386 for source in registry.sourceFormats {387 for destination in resolver.capabilities(for: source).chainedDestinations {388 guard let plan = try? planner.plan(from: source, to: destination), plan.isChained, let middle = plan.intermediates.first else { continue }389 pairs.append((.init(source: source.rawValue, destination: destination.rawValue, through: middle.rawValue, engines: plan.engines, fidelity: plan.fidelity.rawValue), plan))390 }391 }392 let groups = Dictionary(grouping: pairs.filter { primaryFormats.contains(FormatID($0.pair.source)) }, by: { $0.pair.engines.joined(separator: " + ") })393 var sample: [(pair: SweepReport.ChainPair, plan: ConversionPlan)] = []394 var cursor = 0395 let keys = groups.keys.sorted()396 while sample.count < chainSample, !keys.isEmpty {397 var added = false398 for key in keys {399 guard let list = groups[key], sample.count < chainSample else { continue }400 // Spread the picks over the group instead of taking its first rows.401 let stride = max(1, list.count / max(1, (chainSample / keys.count) + 1))402 let position = cursor * stride403 if position < list.count { sample.append(list[position]); added = true }404 }405 cursor += 1406 if !added { break }407 }408 io.err("Chained plans: \(pairs.count) pair(s) bridged by the planner; running a sample of \(sample.count).")409 await withTaskGroup(of: SweepResult.self) { group in410 for (index, item) in sample.enumerated() {411 let source = FormatID(item.pair.source), destination = FormatID(item.pair.destination)412 let id = "chain:\(item.pair.source)>\(item.pair.through)>\(item.pair.destination)"413 if let previous = collector.result(id), previous.status == "pass" { chainResults.append(previous); continue }414 group.addTask {415 await execute(id: id, kind: "chain", source: source, destination: destination, operation: nil, engines: item.pair.engines, engineID: nil, multi: false, index: index)416 }417 }418 for await result in group {419 _ = collector.record(result)420 chainResults.append(result)421 if result.status == "refused" {422 screen.error("REFUSED (content: \(result.refusalReason ?? "?")) \(result.id): \(result.error ?? "")")423 } else if result.status != "pass" {424 screen.error("\(result.status.uppercased()) \(result.id): \(result.error ?? result.note ?? "")")425 }426 }427 }428 collector.save()429 var totals = SweepReport.Totals()430 for result in chainResults { add(result, to: &totals) }431 chainReport = .init(totalPairs: pairs.count, byEngines: Dictionary(grouping: pairs, by: { $0.pair.engines.joined(separator: " + ") }).mapValues(\.count),432 sampled: sample.count, totals: totals, pairs: pairs.map(\.pair))433 }434 watchdog.cancel()435 await queue.waitUntilIdle()436 if !keep { try? fm.removeItem(at: outRoot) }437438 // Report.439 let all = collector.results440 let results = paths.compactMap { all[$0.id] }441 var totals = SweepReport.Totals()442 var byEngine: [String: SweepReport.Totals] = [:]443 for result in results {444 add(result, to: &totals)445 add(result, to: &byEngine[result.engines.first ?? "?", default: .init()])446 }447 let allFixtures = await store.all448 var detectorNotes: [SweepReport.DetectorNote] = []449 for fixture in allFixtures {450 let format = FormatID(fixture.format)451 let detection = FileTypeDetector.detect(fixture.url)452 if detection.format != format {453 detectorNotes.append(.init(format: format.rawValue, detectedAs: detection.format?.rawValue, evidence: detection.evidence, confidence: detection.confidence.rawValue,454 fixtureOrigin: fixture.origin, acceptedAsEquivalent: detection.format.map { FormatEquivalence.accepts(detected: $0, as: format) } ?? false))455 }456 }457 // Which engine inspects each format, on the primary fixtures.458 var inspections: [SweepReport.InspectionNote] = []459 for format in primaryFormats.sorted() {460 guard let fixture = await store.primary(format) else { continue }461 if FormatTypeCheck.isFolderFixture(fixture.url, format: format) { continue }462 let inspected = await service.inspector.inspect(fixture.url)463 inspections.append(.init(format: format.rawValue, engine: inspected.engineID, error: inspected.engineError,464 engineFields: inspected.inspection.fields.filter { $0.group != "File" }.count))465 }466 let prefix = "\(ProcessInfo.processInfo.processIdentifier)-"467 let leftovers = service.workRootLeftovers().filter { $0.hasPrefix(prefix) }468 let formatter = ISO8601DateFormatter()469 let environment = SweepReport.Environment(470 date: formatter.string(from: started), tool: "orix-convert \(CLI.version)", machine: machine(),471 system: ProcessInfo.processInfo.operatingSystemVersionString, loadAverageStart: loadStart, loadAverageEnd: loadAverage(), jobs: jobs, heavyJobs: heavy,472 timeoutSeconds: timeout, path: io.environment["PATH"] ?? "", toolsEnvironment: ProcessInfo.processInfo.environment["ORIX_CONVERT_TOOLS"].map { value in repository.map { value.replacingOccurrences(of: $0.path, with: "<repo>") } ?? value },473 wallSeconds: Date().timeIntervalSince(started), filter: filterText.isEmpty ? nil : filterText)474 let report = SweepReport(environment: environment, registryFingerprint: fingerprint, totals: totals, byEngine: byEngine,475 fixtures: allFixtures, formatsWithoutFixture: Dictionary(uniqueKeysWithValues: missing.map { ($0.key.rawValue, $0.value) }),476 detectorNotes: detectorNotes, inspections: inspections, chains: chainReport, temporaryLeftovers: leftovers,477 results: results, chainResults: chainResults.sorted { $0.id < $1.id })478 let suffix = filterText.isEmpty ? "" : "-partial"479 let jsonURL = reportDirectory.appendingPathComponent("e2e-matrix\(suffix).json"), markdownURL = reportDirectory.appendingPathComponent("e2e-matrix\(suffix).md")480 try Data((try Shared.json(report) + "\n").utf8).write(to: jsonURL, options: .atomic)481 try Data(markdown(report, workDirectory: workDirectory).utf8).write(to: markdownURL, options: .atomic)482 io.out("End-to-end sweep: \(totals.total) path(s) — \(totals.pass) passed, \(totals.fail) failed, \(totals.crash) crashed, \(totals.timeout) timed out, \(totals.skip) skipped\(totals.cancelled > 0 ? ", \(totals.cancelled) cancelled" : "") — \(Shared.seconds(environment.wallSeconds))")483 if let chains = chainReport {484 io.out("Chained plans: \(chains.totalPairs) pair(s) exist; sample of \(chains.sampled): \(chains.totals.pass) passed, \(chains.totals.fail + chains.totals.timeout) failed, \(chains.totals.refusedContent) refused: content, \(chains.totals.skip) skipped.")485 }486 io.out("Temporary work folders left behind: \(leftovers.isEmpty ? "none" : leftovers.joined(separator: ", "))")487 io.out("Report: \(markdownURL.path)")488 if InterruptCenter.shared.isInterrupted { return .interrupted }489 let chainFailures = (chainReport?.totals.fail ?? 0) + (chainReport?.totals.timeout ?? 0)490 return totals.fail + totals.crash + totals.timeout + chainFailures > 0 || !leftovers.isEmpty ? .conversionFailed : .success491 }492493 /// Runs one path alone in a child process (`matrix --verify --only-id`). A child killed by a signal is a crash.494 static func runInChild(path: ConversionPath, workDirectory: URL, timeout: Double) async -> SweepResult {495 var result = SweepResult(id: path.id, kind: "path", source: path.source.rawValue, destination: path.destination.rawValue, operation: path.operation,496 engines: [path.engine], status: "fail", seconds: nil, error: nil, errorKind: nil, failedCheck: nil, detectedAs: nil,497 explicitSourceFormat: false, fixtureOrigin: nil, fixtureVariant: nil, primaryFixtureError: nil, primaryFixtureOrigin: nil,498 primaryErrorKind: nil, variantsTried: nil, crashedWith: nil, options: [:], checks: 0, warnings: [], outputs: 0, outputBytes: nil, note: nil, refusalReason: nil)499 guard let executable = ToolEnvironment.executableURL() else { result.error = "The executable could not be located for the child run."; return result }500 let process = Process()501 process.executableURL = executable502 process.arguments = ["matrix", "--verify", "--work-dir", workDirectory.path, "--report-dir", workDirectory.appendingPathComponent("child-report").path,503 "--only-id", path.id, "--timeout", String(timeout), "--no-chains"]504 let pipe = Pipe()505 process.standardOutput = pipe506 process.standardError = FileHandle.nullDevice507 process.standardInput = FileHandle.nullDevice508 let started = Date()509 let outcome: (Data, Process.TerminationReason, Int32)? = await withCheckedContinuation { continuation in510 DispatchQueue.global().async {511 do {512 try process.run()513 let data = pipe.fileHandleForReading.readDataToEndOfFile()514 process.waitUntilExit()515 continuation.resume(returning: (data, process.terminationReason, process.terminationStatus))516 } catch {517 continuation.resume(returning: nil)518 }519 }520 }521 guard let (data, reason, status) = outcome else { result.error = "The child process could not be started."; return result }522 if reason == .uncaughtSignal {523 result.status = "crash"524 result.seconds = Date().timeIntervalSince(started)525 result.error = "The process was killed by signal \(status) while converting this path alone."526 result.errorKind = "crash"527 return result528 }529 if let decoded = try? JSONDecoder().decode(SweepResult.self, from: data) { return decoded }530 result.error = "The child process ended with status \(status) without a result."531 return result532 }533534 static func add(_ result: SweepResult, to totals: inout SweepReport.Totals) {535 totals.total += 1536 switch result.status {537 case "pass":538 totals.pass += 1539 if (result.fixtureVariant ?? 0) > 0 { totals.passOnVariantOnly += 1 }540 case "fail": totals.fail += 1541 case "crash": totals.crash += 1542 case "skip": totals.skip += 1543 case "timeout": totals.timeout += 1544 case "refused": totals.refusedContent += 1545 default: totals.cancelled += 1546 }547 }548549 static func markdown(_ report: SweepReport, workDirectory: URL) -> String {550 func cell(_ text: String) -> String { text.replacingOccurrences(of: "|", with: "\\|").replacingOccurrences(of: "\n", with: " ") }551 func seconds(_ value: Double?) -> String { value.map { String(format: "%.2f", locale: Locale(identifier: "en_US_POSIX"), $0) } ?? "—" }552 let e = report.environment, t = report.totals553 var out = "# ORIX Convert — end-to-end sweep of the conversion matrix\n\n"554 out += "> Generated file — do not edit. Written by `orix-convert matrix --verify` (`Scripts/matrix.sh --verify`).\n\n"555 out += "Every declared path was run as a user would run it: the file goes through the detector, the planner, the queue, the engine, the generic\n"556 out += "validation, the engine's own validation and the safe writer, into a temporary folder. Fixtures are real files: \(FixtureStore.seedFormats.count) seeds generated in code\n"557 out += "(\(FixtureStore.seedFormats.map(\.rawValue).joined(separator: ", "))); every other format was obtained by converting a fixture through the engines.\n"558 out += "Each path runs with its default settings and is forced onto the engine that declares it (so a path is tested even when the planner\n"559 out += "would prefer another engine for the same pair).\n\n"560 out += "| | |\n|---|---|\n| Date | \(e.date) |\n| Tool | \(e.tool) |\n| Machine | \(e.machine) |\n| System | \(e.system) |\n"561 out += "| Load average (1, 5, 15 min) at start | \(e.loadAverageStart.map { String($0) }.joined(separator: ", ")) |\n| Load average at end | \(e.loadAverageEnd.map { String($0) }.joined(separator: ", ")) |\n"562 out += "| Concurrency | \(e.jobs) jobs, \(e.heavyJobs) heavy |\n| Time limit per path | \(Int(e.timeoutSeconds)) s from job start |\n| PATH of the run | `\(e.path)` |\n"563 out += "| ORIX_CONVERT_TOOLS | \(e.toolsEnvironment.map { "`\($0)`" } ?? "not set") |\n| Wall time of this invocation | \(seconds(e.wallSeconds)) s (resumed runs only count the last invocation) |\n"564 out += "| Registry fingerprint | `\(report.registryFingerprint)` |\n"565 if let filter = e.filter { out += "| Filter | \(filter) — PARTIAL RUN |\n" }566 out += "\nSettings given to operations whose defaults ask for nothing (every other path ran with its defaults): "567 out += operationOptions.sorted { $0.key < $1.key }.map { op, values in "`\(op)` → " + values.sorted { $0.key < $1.key }.map { "`\($0.key)=\(Shared.optionValueText($0.value))`" }.joined(separator: " ") }.joined(separator: "; ") + ".\n"568 out += "\n## Totals\n\n| Paths | Passed | Failed | Crashed | Timed out | Skipped (no fixture) | Cancelled |\n|---:|---:|---:|---:|---:|---:|---:|\n"569 out += "| \(t.total) | \(t.pass) | \(t.fail) | \(t.crash) | \(t.timeout) | \(t.skip) | \(t.cancelled) |\n\n"570 out += "Of the \(t.pass) passes, \(t.passOnVariantOnly) needed an alternative fixture (listed below with the error met on the primary one).\n\n"571 out += "| Engine | Paths | Passed | Failed | Crashed | Timed out | Skipped |\n|---|---:|---:|---:|---:|---:|---:|\n"572 for (engine, c) in report.byEngine.sorted(by: { $0.key < $1.key }) {573 out += "| \(engine) | \(c.total) | \(c.pass) | \(c.fail) | \(c.crash) | \(c.timeout) | \(c.skip) |\n"574 }575 out += "\nTemporary work folders left behind after the run: \(report.temporaryLeftovers.isEmpty ? "none" : report.temporaryLeftovers.joined(separator: ", ")).\n\n"576577 let suspects = report.results.filter { $0.crashedWith != nil }578 if !suspects.isEmpty {579 out += "## Process crashes (\(suspects.count) path(s) involved)\n\n"580 out += "An invocation of the sweep was killed while these paths were being converted at the same time. Each was then re-run alone in a child process;\n"581 out += "the result of that re-run is the one counted.\n\n| Path | Re-run alone | Running at the same time when the process died |\n|---|---|---|\n"582 for r in suspects { out += "| `\(r.id)` | \(r.status)\(r.error.map { " — " + cell($0) } ?? "") | \((r.crashedWith ?? []).map { "`\($0)`" }.joined(separator: ", ")) |\n" }583 out += "\n"584 }585586 let failures = report.results.filter { $0.status == "fail" || $0.status == "timeout" || $0.status == "crash" }587 out += "## Failures (\(failures.count))\n\n"588 if failures.isEmpty {589 out += "None.\n\n"590 } else {591 out += "| Path | Result | Error kind | Failed check | Seconds | Error |\n|---|---|---|---|---:|---|\n"592 for f in failures {593 out += "| `\(f.id)` | \(f.status) | \(f.errorKind ?? "—") | \(f.failedCheck ?? "—") | \(seconds(f.seconds)) | \(cell(f.error ?? "")) (fixture: `\(f.fixtureOrigin ?? "?")`, \(f.variantsTried ?? 1) fixture(s) tried) |\n"594 }595 out += "\n"596 }597598 let variants = report.results.filter { $0.status == "pass" && ($0.fixtureVariant ?? 0) > 0 }599 out += "## Passed only on an alternative fixture (\(variants.count))\n\n"600 if variants.isEmpty {601 out += "None.\n\n"602 } else {603 out += "These paths failed on the primary fixture of their source format and passed on another real file of that format. They are counted as passed;\n"604 out += "the first error is kept here because it is either a legitimate refusal of that particular content or an integration defect.\n\n"605 out += "| Path | Primary fixture | Error on the primary | Passed with |\n|---|---|---|---|\n"606 for v in variants {607 out += "| `\(v.id)` | `\(v.primaryFixtureOrigin ?? "?")` | \(v.primaryErrorKind ?? "—"): \(cell(v.primaryFixtureError ?? "")) | `\(v.fixtureOrigin ?? "?")` |\n"608 }609 out += "\n"610 }611612 let skipped = report.results.filter { $0.status == "skip" }613 out += "## Skipped paths (\(skipped.count))\n\n"614 if skipped.isEmpty {615 out += "None: a fixture exists for every source format.\n\n"616 } else {617 out += "A path is skipped only when no real file of its source format could be produced.\n\n| Source format | Paths skipped | Why no fixture |\n|---|---:|---|\n"618 for (format, list) in Dictionary(grouping: skipped, by: \.source).sorted(by: { $0.key < $1.key }) {619 out += "| \(format) | \(list.count) | \(cell(report.formatsWithoutFixture[format] ?? "—")) |\n"620 }621 out += "\n"622 }623624 out += "## Detector and engine outputs\n\n"625 if report.detectorNotes.isEmpty {626 out += "The detector names every fixture as the format it is meant to be.\n\n"627 } else {628 out += "Fixtures the detector does not name as the format they stand for. Their paths were run with the source format given\n"629 out += "explicitly (a user dropping such a file would get the detected format's conversions instead).\n\n"630 out += "| Fixture of | Detected as | Confidence | Evidence | Listed as equivalent | Fixture made by |\n|---|---|---|---|---|---|\n"631 for n in report.detectorNotes {632 out += "| \(n.format) | \(n.detectedAs ?? "unrecognised") | \(n.confidence) | \(cell(n.evidence)) | \(n.acceptedAsEquivalent ? "yes" : "no") | `\(n.fixtureOrigin)` |\n"633 }634 out += "\n"635 }636637 out += "## Inspection\n\n`inspect` was run on the primary fixture of every format (the engine is the one the registry designates).\n\n"638 out += "| Inspected by | Formats |\n|---|---|\n"639 for (engine, list) in Dictionary(grouping: report.inspections.filter { $0.error == nil }, by: { $0.engine ?? "no engine (generic file facts only)" }).sorted(by: { $0.key < $1.key }) {640 out += "| \(engine) | \(list.map(\.format).joined(separator: ", ")) |\n"641 }642 let inspectionErrors = report.inspections.filter { $0.error != nil }643 if inspectionErrors.isEmpty {644 out += "\nNo inspection failed.\n\n"645 } else {646 out += "\n| Format | Inspection error |\n|---|---|\n"647 for note in inspectionErrors { out += "| \(note.format) | \(cell(note.error ?? "")) |\n" }648 out += "\n"649 }650651 if let chains = report.chains {652 out += "## Chained plans\n\n"653 out += "Pairs with no direct path that the planner bridges with two steps: **\(chains.totalPairs)**. A sample of \(chains.sampled) was run end to end "654 out += "(spread over the engine combinations): \(chains.totals.pass) passed (\(chains.totals.passOnVariantOnly) of them on an alternative fixture), \(chains.totals.fail) failed, \(chains.totals.timeout) timed out, "655 out += "\(chains.totals.refusedContent) refused: content, \(chains.totals.skip) skipped.\n\n"656 out += "A chain is first run on the primary fixture of its source format, then on the other real files of that format (for instance an SVG with\n"657 out += "text elements, a NetCDF grid with `lat` / `lon` coordinate variables). \"refused: content\" means that no fixture could satisfy the second step and\n"658 out += "that the engine explained the refusal by the content of the file (\(ContentRefusal.rules.map(\.reason).joined(separator: ", "))): it is reported, not counted as a failure.\n"659 out += "Any other error is a failure.\n\n"660 out += "| Engines (step 1 + step 2) | Pairs |\n|---|---:|\n"661 for (key, count) in chains.byEngines.sorted(by: { $0.key < $1.key }) { out += "| \(key) | \(count) |\n" }662 out += "\n| Chain run | Result | Fixture | Seconds | Checks | Error |\n|---|---|---|---:|---:|---|\n"663 for r in report.chainResults {664 let status = r.status == "refused" ? "refused: content (\(r.refusalReason ?? "?"))" : r.status665 let fixture = (r.fixtureVariant ?? 0) > 0 ? "`\(r.fixtureOrigin ?? "?")` (the primary fixture was refused: \(cell(r.primaryFixtureError ?? "")))" : "primary"666 out += "| \(r.note ?? r.id) (\(r.engines.joined(separator: " + "))) | \(status) | \(fixture) | \(seconds(r.seconds)) | \(r.checks) | \(cell(r.error ?? "")) |\n"667 }668 out += "\nThe complete list of chained pairs is in `e2e-matrix.json` (`chains.pairs`).\n\n"669 }670671 let slow = report.results.filter { $0.seconds != nil }.sorted { ($0.seconds ?? 0) > ($1.seconds ?? 0) }.prefix(15)672 out += "## Slowest paths\n\n| Path | Seconds | Result |\n|---|---:|---|\n"673 for r in slow { out += "| `\(r.id)` | \(seconds(r.seconds)) | \(r.status) |\n" }674 out += "\nTimes are job durations on a shared machine under the load shown above; they are not benchmarks.\n\n"675676 out += "## Fixtures\n\n| Format | Origin |\n|---|---|\n"677 for f in report.fixtures { out += "| \(f.format) | \(f.origin == "seed" ? "seed (generated in code)" : "`\(f.origin)`") |\n" }678 out += "\n## Every path\n\n| Path | Result | Seconds | Checks | Warnings | Outputs |\n|---|---|---:|---:|---|---:|\n"679 for r in report.results {680 out += "| `\(r.id)` | \(r.status)\(r.explicitSourceFormat ? " (format given)" : "") | \(seconds(r.seconds)) | \(r.checks) | \(r.warnings.isEmpty ? "—" : cell(Array(Set(r.warnings)).sorted().joined(separator: ", "))) | \(r.outputs) |\n"681 }682 return out683 }684}685