SPB Git

Repository Public License

Sweep.swift 44.3 KB 685 lines swift

1// ORIX Convert — Packages/ConvertKit/Sources/ConvertCLI/Sweep.swift2// Description: The cross-engine end-to-end sweep (`orix-convert matrix --verify`). Every declared path is run exactly3//              as a user would run it — detector, planner, queue, engine, generic and engine validation, safe writer —4//              on a real fixture, with bounded concurrency, a per-path time limit and resumable state.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import ConvertCore8import ConvertKit910struct SweepResult: Codable, Sendable {11    var id: String12    var kind: String               // "path" or "chain"13    var source: String, destination: String14    var operation: String?15    var engines: [String]16    var status: String             // pass, fail, skip, timeout, cancelled, crash; chains only: refused (content)17    var seconds: Double?18    var error: String?19    var errorKind: String?20    var failedCheck: String?21    /// What the detector says the fixture is, when that differs from the path's source format.22    var detectedAs: String?23    /// The job was given the source format explicitly because the detector names another format for the fixture.24    var explicitSourceFormat: Bool25    var fixtureOrigin: String?26    /// 0 = the primary fixture of the source format; above, an alternative real file of the same format.27    var fixtureVariant: Int?28    /// When the path passed on a variant only: what happened on the primary fixture.29    var primaryFixtureError: String?30    var primaryFixtureOrigin: String?31    var primaryErrorKind: String?32    /// For a failure: how many different fixtures were tried.33    var variantsTried: Int?34    /// Set when an earlier invocation of the sweep crashed while this path was running: the paths that were running35    /// with it. The result recorded here is the one of the re-run, alone, in a child process.36    var crashedWith: [String]?37    var options: [String: String]38    var checks: Int39    var warnings: [String]40    var outputs: Int41    var outputBytes: Int64?42    var note: String?43    /// For a chain reported as "refused": which content-dependent refusal the engine explained (`ContentRefusal`).44    var refusalReason: String?45}4647struct SweepState: Codable {48    var fingerprint: String49    var results: [String: SweepResult]50}5152struct SweepReport: Codable {53    struct Environment: Codable {54        let date: String, tool: String, machine: String, system: String55        let loadAverageStart: [Double], loadAverageEnd: [Double]56        let jobs: Int, heavyJobs: Int, timeoutSeconds: Double57        let path: String58        let toolsEnvironment: String?59        let wallSeconds: Double60        let filter: String?61    }62    struct Totals: Codable {63        var total = 0, pass = 0, passOnVariantOnly = 0, fail = 0, crash = 0, skip = 0, timeout = 0, cancelled = 064        /// Chained plans only: the engine refused the fixture for a reason that depends on its content.65        var refusedContent = 066    }67    struct DetectorNote: Codable { let format: String; let detectedAs: String?; let evidence: String; let confidence: String; let fixtureOrigin: String; let acceptedAsEquivalent: Bool }68    struct InspectionNote: Codable { let format: String; let engine: String?; let error: String?; let engineFields: Int }69    struct ChainPair: Codable { let source: String; let destination: String; let through: String; let engines: [String]; let fidelity: String }70    struct Chains: Codable {71        let totalPairs: Int72        let byEngines: [String: Int]73        let sampled: Int74        let totals: Totals75        let pairs: [ChainPair]76    }77    let environment: Environment78    let registryFingerprint: String79    let totals: Totals80    let byEngine: [String: Totals]81    let fixtures: [Fixture]82    let formatsWithoutFixture: [String: String]83    let detectorNotes: [DetectorNote]84    let inspections: [InspectionNote]85    let chains: Chains?86    let temporaryLeftovers: [String]87    let results: [SweepResult]88    let chainResults: [SweepResult]89}9091enum FormatTypeCheck {92    /// The folder seed is a directory: there is nothing to inspect in it.93    static func isFolderFixture(_ url: URL, format: FormatID) -> Bool { format == .folder }94}9596enum Sweep {97    /// Options a named operation needs to do anything at all (its defaults ask for "nothing"). Everything else runs98    /// with the path's defaults, as a user who changes no setting.99    static let operationOptions: [String: [String: OptionValue]] = [100        "pdf.extract-pages": ["pages": .string("1-2")],101        "pdf.delete-pages": ["pages": .string("1")],102        "pdf.set-metadata": ["metadata.title": .string("ORIX Convert sweep")],103    ]104105    static func loadAverage() -> [Double] {106        var values = [Double](repeating: 0, count: 3)107        getloadavg(&values, 3)108        return values.map { ($0 * 100).rounded() / 100 }109    }110111    static func machine() -> String {112        func sysctl(_ name: String) -> String {113            var size = 0114            sysctlbyname(name, nil, &size, nil, 0)115            var buffer = [CChar](repeating: 0, count: max(1, size))116            sysctlbyname(name, &buffer, &size, nil, 0)117            return String(cString: buffer)118        }119        let memory = ProcessInfo.processInfo.physicalMemory / (1 << 30)120        return "\(sysctl("machdep.cpu.brand_string")), \(ProcessInfo.processInfo.processorCount) cores, \(memory) GB (\(sysctl("hw.model")))"121    }122123    final class Collector: @unchecked Sendable {124        private let lock = NSLock()125        private var state: SweepState126        private let url: URL127        private var unsaved = 0128        var timedOut = Set<UUID>()129130        init(state: SweepState, url: URL) { self.state = state; self.url = url }131132        func record(_ result: SweepResult) -> (done: Int, pass: Int, fail: Int, skip: Int) {133            lock.lock(); defer { lock.unlock() }134            state.results[result.id] = result135            unsaved += 1136            if unsaved >= 20 { saveLocked() }137            let values = state.results.values138            return (values.count, values.filter { $0.status == "pass" }.count, values.filter { $0.status == "fail" || $0.status == "timeout" || $0.status == "crash" }.count,139                    values.filter { $0.status == "skip" }.count)140        }141142        private var jobPaths: [UUID: String] = [:]143        private var lastInFlight: [String] = []144        var inFlightURL: URL { url.deletingLastPathComponent().appendingPathComponent("in-flight.json") }145146        func bind(_ job: UUID, to path: String) { lock.lock(); jobPaths[job] = path; lock.unlock() }147148        /// Journal of the paths being converted right now. If the process dies (an engine crash takes the whole149        /// process with it), the next invocation finds the suspects here.150        func noteActive(_ jobs: [UUID]) {151            lock.lock()152            let paths = jobs.compactMap { jobPaths[$0] }.sorted()153            let changed = paths != lastInFlight154            lastInFlight = paths155            lock.unlock()156            if changed, let data = try? JSONEncoder().encode(paths) { try? data.write(to: inFlightURL, options: .atomic) }157        }158159        func clearInFlight() { try? FileManager.default.removeItem(at: inFlightURL) }160161        func markTimedOut(_ id: UUID) { lock.lock(); timedOut.insert(id); lock.unlock() }162        func didTimeOut(_ id: UUID) -> Bool { lock.lock(); defer { lock.unlock() }; return timedOut.contains(id) }163        func result(_ id: String) -> SweepResult? { lock.lock(); defer { lock.unlock() }; return state.results[id] }164        var results: [String: SweepResult] { lock.lock(); defer { lock.unlock() }; return state.results }165        func save() { lock.lock(); saveLocked(); lock.unlock() }166167        private func saveLocked() {168            unsaved = 0169            if let text = try? Shared.json(state) { try? Data(text.utf8).write(to: url, options: .atomic) }170        }171    }172173    static func run(_ arguments: ParsedArguments, io: CommandIO, repository: URL?) async throws -> ExitCode {174        let started = Date()175        let loadStart = loadAverage()176        let jobs = try arguments.int("jobs", in: 1...32) ?? 6177        let heavy = try arguments.int("heavy-jobs", in: 1...32) ?? max(2, jobs / 2)178        let timeout = try arguments.double("timeout", in: 5...7200) ?? 300179        let chainSample = arguments.flag("no-chains") ? 0 : (try arguments.int("chains", in: 0...2000) ?? 40)180        let workDirectory = arguments.value("work-dir").map(Shared.fileURL) ?? FileManager.default.temporaryDirectory.appendingPathComponent("orix-convert-e2e", isDirectory: true)181        guard let reportDirectory = arguments.value("report-dir").map(Shared.fileURL) ?? repository?.appendingPathComponent("docs/reports") else {182            throw UsageError("matrix --verify needs --report-dir when it does not run from the repository.")183        }184        let fm = FileManager.default185        if arguments.flag("fresh") { try? fm.removeItem(at: workDirectory) }186        try fm.createDirectory(at: workDirectory, withIntermediateDirectories: true)187        try fm.createDirectory(at: reportDirectory, withIntermediateDirectories: true)188189        let service = ConversionService(configuration: .init(maxConcurrentJobs: jobs, maxHeavyJobs: heavy, recordHistory: false))190        let registry = service.registry191        let document = MatrixBuilder.build(registry: registry, repository: repository)192        let fingerprint = document.identity.registryFingerprint193        let engineFilter = arguments.value("engine")194        let from = try arguments.value("from").map { try Shared.format($0, option: "--from") }195        let to = try arguments.value("to").map { try Shared.format($0, option: "--to") }196        let paths = registry.paths(from: from, to: to, engine: engineFilter)197        let filterText = [engineFilter.map { "engine=\($0)" }, from.map { "from=\($0)" }, to.map { "to=\($0)" }].compactMap { $0 }.joined(separator: " ")198199        let stateURL = workDirectory.appendingPathComponent("state.json")200        var state = SweepState(fingerprint: fingerprint, results: [:])201        if let data = try? Data(contentsOf: stateURL), let saved = try? JSONDecoder().decode(SweepState.self, from: data) {202            if saved.fingerprint == fingerprint {203                state = saved204            } else {205                io.err("The registry changed since the saved state (fingerprint differs): results of paths that still exist are kept.")206                let ids = Set(registry.paths.map(\.id))207                state.results = saved.results.filter { ids.contains($0.key) || $0.value.kind == "chain" }208            }209        }210        if arguments.flag("retry-failed") { state.results = state.results.filter { $0.value.status == "pass" } }211        // Skips are always re-evaluated: a fixture may exist now.212        state.results = state.results.filter { $0.value.status != "skip" && $0.value.status != "cancelled" }213        let collector = Collector(state: state, url: stateURL)214215        // Fixtures.216        io.err("Preparing fixtures in \(workDirectory.path) …")217        let store = FixtureStore(directory: workDirectory.appendingPathComponent("fixtures", isDirectory: true), service: service, log: { io.err("  " + $0) })218        await store.loadManifest()219        let childMode = arguments.value("only-id") != nil220        if !childMode { await store.build(for: Set(registry.paths.map(\.source))) }221        let missing = await store.missing222        let primaryFormats = Set(await store.fixtures.keys)223        io.err("Fixtures: \(primaryFormats.count) format(s); \(missing.count) source format(s) without a fixture.")224225        // Time limit: counted from the start of each job.226        let queue = service.queue227        let watchdog = Task {228            while !Task.isCancelled {229                try? await Task.sleep(nanoseconds: 500_000_000)230                let now = Date()231                let active = await queue.snapshot().jobs.filter { $0.state.isActive }232                collector.noteActive(active.map(\.id))233                for job in active where !job.cancelRequested {234                    if let start = job.startedAt, now.timeIntervalSince(start) > timeout {235                        collector.markTimedOut(job.id)236                        await queue.cancel(job.id)237                    }238                }239            }240        }241        let interrupt = InterruptCenter.shared.register { Task { await queue.cancelAll() } }242        defer { InterruptCenter.shared.unregister(interrupt) }243244        let outRoot = workDirectory.appendingPathComponent("out", isDirectory: true)245        if !childMode { try? fm.removeItem(at: outRoot) }246        let keep = arguments.flag("keep-outputs")247        let screen = Screen(io: io, showsProgress: true)248        let totalPlanned = paths.count249250        @Sendable func execute(id: String, kind: String, source: FormatID, destination: FormatID, operation: String?, engines: [String], engineID: String?,251                               multi: Bool, index: Int) async -> SweepResult {252            var result = SweepResult(id: id, kind: kind, source: source.rawValue, destination: destination.rawValue, operation: operation, engines: engines,253                                     status: "skip", seconds: nil, error: nil, errorKind: nil, failedCheck: nil, detectedAs: nil, explicitSourceFormat: false,254                                     fixtureOrigin: nil, fixtureVariant: nil, primaryFixtureError: nil, primaryFixtureOrigin: nil, primaryErrorKind: nil, variantsTried: nil, crashedWith: nil, options: [:], checks: 0, warnings: [], outputs: 0, outputBytes: nil, note: nil, refusalReason: nil)255            guard primaryFormats.contains(source) else {256                result.note = "No fixture of \(source.rawValue): \(missing[source] ?? "not produced")."257                return result258            }259            // The primary fixture first; when the path cannot work on it, other real files of the same format260            // (seed variants, the same format written by another path). A pass on a variant is reported as such,261            // with the primary's error.262            var first: SweepResult?263            var variant = 0264            while let fixture = await store.fixture(source, index: variant, hint: (destination, operation)) {265                var attempt = result266                attempt.fixtureOrigin = fixture.origin267                attempt.fixtureVariant = variant268                let detection = FileTypeDetector.detect(fixture.url)269                let explicit = detection.format != source270                if explicit { attempt.detectedAs = detection.format?.rawValue ?? "unrecognised"; attempt.explicitSourceFormat = true }271                var sources = [fixture.url]272                if multi {273                    let second = fixture.url.deletingLastPathComponent().appendingPathComponent("fixture-b" + (fixture.url.pathExtension.isEmpty ? "" : "." + fixture.url.pathExtension))274                    if !FileManager.default.fileExists(atPath: second.path) { try? FileManager.default.copyItem(at: fixture.url, to: second) }275                    if FileManager.default.fileExists(atPath: second.path) { sources.append(second) }276                }277                let options = ConversionOptions(operation.flatMap { operationOptions[$0] } ?? [:])278                attempt.options = options.values.mapValues(Shared.optionValueText)279                let output = outRoot.appendingPathComponent("\(kind)-\(index)-\(variant)", isDirectory: true)280                let request = ConversionJobRequest(sources: sources, sourceFormat: explicit ? source : nil, destinationFormat: destination, operation: operation,281                                                   options: options, engineID: engineID, destination: OutputDestination(directory: output, conflictPolicy: .rename),282                                                   allowChaining: kind == "chain", recordInHistory: false)283                let jobID = await queue.enqueue(request)284                collector.bind(jobID, to: id)285                let job = await queue.waitForJob(jobID)286                if !keep { try? FileManager.default.removeItem(at: output) }287                guard let job else { attempt.status = "fail"; attempt.error = "The job disappeared from the queue."; return attempt }288                attempt.seconds = job.duration289                attempt.checks = job.checks.count290                attempt.warnings = job.warnings.map(\.code)291                attempt.outputs = job.outputs.count292                attempt.outputBytes = job.bytesAfter293                if kind == "chain", let plan = job.plan { attempt.engines = plan.engines; attempt.note = ([plan.source] + plan.steps.map(\.destination)).map(\.rawValue).joined(separator: " → ") }294                switch job.state {295                case .finished:296                    attempt.status = "pass"297                    if job.outputs.isEmpty { attempt.status = "fail"; attempt.error = "The job finished without any output."; attempt.errorKind = "outputInvalid" }298                case .cancelled:299                    if collector.didTimeOut(jobID) {300                        attempt.status = "timeout"; attempt.error = "Not finished \(Int(timeout)) s after it started; cancelled."301                    } else {302                        attempt.status = "cancelled"303                    }304                default:305                    attempt.status = "fail"306                    attempt.error = job.error ?? "Ended in state \(job.state.rawValue)."307                    attempt.errorKind = job.failure?.kind308                    attempt.failedCheck = job.checks.first(where: { !$0.passed })?.name309                }310                _ = await queue.remove(jobID)311                if attempt.status == "pass" {312                    if let first { attempt.primaryFixtureError = first.error; attempt.primaryFixtureOrigin = first.fixtureOrigin; attempt.primaryErrorKind = first.errorKind }313                    return attempt314                }315                if first == nil { first = attempt }316                if attempt.status != "fail" { break }317                variant += 1318            }319            if var first {320                first.variantsTried = variant + 1321                // A chain hands the second step whatever the first one wrote. When no fixture can satisfy it and the322                // engine says why in terms of the content, that is a refusal of the content, not a failure of the plan.323                if kind == "chain", first.status == "fail", let reason = ContentRefusal.reason(kind: first.errorKind, message: first.error) {324                    first.status = "refused"325                    first.refusalReason = reason326                }327                return first328            }329            return result330        }331332        // Child mode: one path, alone in this process; the result goes to standard output as JSON.333        if let only = arguments.value("only-id") {334            guard let path = registry.paths.first(where: { $0.id == only }) else { throw UsageError("--only-id: no path '\(only)'.") }335            let result = await execute(id: path.id, kind: "path", source: path.source, destination: path.destination, operation: path.operation, engines: [path.engine],336                                       engineID: path.engine, multi: path.acceptsMultipleSources, index: Int(ProcessInfo.processInfo.processIdentifier))337            watchdog.cancel()338            io.out(try Shared.json(result))339            return .success340        }341342        // Paths that were running when an earlier invocation died are not run with the others again: each is343        // re-run alone in a child process, so a crash is attributed to one path and cannot take the sweep down.344        var suspects: [String] = []345        if let data = try? Data(contentsOf: collector.inFlightURL), let list = try? JSONDecoder().decode([String].self, from: data) {346            suspects = list.filter { collector.result($0) == nil }347            if !suspects.isEmpty { io.err("The previous invocation ended while \(suspects.count) path(s) were running: \(suspects.joined(separator: ", ")). They are re-run alone, in child processes.") }348        }349        collector.clearInFlight()350        for suspect in suspects {351            guard let path = paths.first(where: { $0.id == suspect }) else { continue }352            var result = await runInChild(path: path, workDirectory: workDirectory, timeout: timeout)353            result.crashedWith = suspects.filter { $0 != suspect }354            _ = collector.record(result)355            screen.error("re-run alone: \(result.status.uppercased()) \(result.id)\(result.error.map { ": \($0)" } ?? "")")356        }357        collector.save()358359        let pending = paths.enumerated().filter { collector.result($0.element.id) == nil }360        io.err("Running \(pending.count) of \(totalPlanned) path(s) (\(totalPlanned - pending.count) already recorded) — \(jobs) at once, \(heavy) heavy, limit \(Int(timeout)) s each.")361        await withTaskGroup(of: SweepResult.self) { group in362            for (index, path) in pending {363                group.addTask {364                    await execute(id: path.id, kind: "path", source: path.source, destination: path.destination, operation: path.operation, engines: [path.engine],365                                  engineID: path.engine, multi: path.acceptsMultipleSources, index: index)366                }367            }368            for await result in group {369                let counts = collector.record(result)370                if result.status == "fail" || result.status == "timeout" {371                    screen.error("\(result.status.uppercased()) \(result.id): \(result.error ?? "")")372                }373                screen.progress("[\(counts.done)] pass \(counts.pass) · fail \(counts.fail) · skip \(counts.skip) — \(result.id)")374            }375        }376        collector.save()377        collector.clearInFlight()378        screen.clear()379380        // Chained plans: every pair the planner bridges, and a sample actually run.381        var chainReport: SweepReport.Chains?382        var chainResults: [SweepResult] = []383        if engineFilter == nil, from == nil, to == nil, !InterruptCenter.shared.isInterrupted {384            let resolver = service.resolver, planner = service.planner385            var pairs: [(pair: SweepReport.ChainPair, plan: ConversionPlan)] = []386            for source in registry.sourceFormats {387                for destination in resolver.capabilities(for: source).chainedDestinations {388                    guard let plan = try? planner.plan(from: source, to: destination), plan.isChained, let middle = plan.intermediates.first else { continue }389                    pairs.append((.init(source: source.rawValue, destination: destination.rawValue, through: middle.rawValue, engines: plan.engines, fidelity: plan.fidelity.rawValue), plan))390                }391            }392            let groups = Dictionary(grouping: pairs.filter { primaryFormats.contains(FormatID($0.pair.source)) }, by: { $0.pair.engines.joined(separator: " + ") })393            var sample: [(pair: SweepReport.ChainPair, plan: ConversionPlan)] = []394            var cursor = 0395            let keys = groups.keys.sorted()396            while sample.count < chainSample, !keys.isEmpty {397                var added = false398                for key in keys {399                    guard let list = groups[key], sample.count < chainSample else { continue }400                    // Spread the picks over the group instead of taking its first rows.401                    let stride = max(1, list.count / max(1, (chainSample / keys.count) + 1))402                    let position = cursor * stride403                    if position < list.count { sample.append(list[position]); added = true }404                }405                cursor += 1406                if !added { break }407            }408            io.err("Chained plans: \(pairs.count) pair(s) bridged by the planner; running a sample of \(sample.count).")409            await withTaskGroup(of: SweepResult.self) { group in410                for (index, item) in sample.enumerated() {411                    let source = FormatID(item.pair.source), destination = FormatID(item.pair.destination)412                    let id = "chain:\(item.pair.source)>\(item.pair.through)>\(item.pair.destination)"413                    if let previous = collector.result(id), previous.status == "pass" { chainResults.append(previous); continue }414                    group.addTask {415                        await execute(id: id, kind: "chain", source: source, destination: destination, operation: nil, engines: item.pair.engines, engineID: nil, multi: false, index: index)416                    }417                }418                for await result in group {419                    _ = collector.record(result)420                    chainResults.append(result)421                    if result.status == "refused" {422                        screen.error("REFUSED (content: \(result.refusalReason ?? "?")) \(result.id): \(result.error ?? "")")423                    } else if result.status != "pass" {424                        screen.error("\(result.status.uppercased()) \(result.id): \(result.error ?? result.note ?? "")")425                    }426                }427            }428            collector.save()429            var totals = SweepReport.Totals()430            for result in chainResults { add(result, to: &totals) }431            chainReport = .init(totalPairs: pairs.count, byEngines: Dictionary(grouping: pairs, by: { $0.pair.engines.joined(separator: " + ") }).mapValues(\.count),432                                sampled: sample.count, totals: totals, pairs: pairs.map(\.pair))433        }434        watchdog.cancel()435        await queue.waitUntilIdle()436        if !keep { try? fm.removeItem(at: outRoot) }437438        // Report.439        let all = collector.results440        let results = paths.compactMap { all[$0.id] }441        var totals = SweepReport.Totals()442        var byEngine: [String: SweepReport.Totals] = [:]443        for result in results {444            add(result, to: &totals)445            add(result, to: &byEngine[result.engines.first ?? "?", default: .init()])446        }447        let allFixtures = await store.all448        var detectorNotes: [SweepReport.DetectorNote] = []449        for fixture in allFixtures {450            let format = FormatID(fixture.format)451            let detection = FileTypeDetector.detect(fixture.url)452            if detection.format != format {453                detectorNotes.append(.init(format: format.rawValue, detectedAs: detection.format?.rawValue, evidence: detection.evidence, confidence: detection.confidence.rawValue,454                                           fixtureOrigin: fixture.origin, acceptedAsEquivalent: detection.format.map { FormatEquivalence.accepts(detected: $0, as: format) } ?? false))455            }456        }457        // Which engine inspects each format, on the primary fixtures.458        var inspections: [SweepReport.InspectionNote] = []459        for format in primaryFormats.sorted() {460            guard let fixture = await store.primary(format) else { continue }461            if FormatTypeCheck.isFolderFixture(fixture.url, format: format) { continue }462            let inspected = await service.inspector.inspect(fixture.url)463            inspections.append(.init(format: format.rawValue, engine: inspected.engineID, error: inspected.engineError,464                                     engineFields: inspected.inspection.fields.filter { $0.group != "File" }.count))465        }466        let prefix = "\(ProcessInfo.processInfo.processIdentifier)-"467        let leftovers = service.workRootLeftovers().filter { $0.hasPrefix(prefix) }468        let formatter = ISO8601DateFormatter()469        let environment = SweepReport.Environment(470            date: formatter.string(from: started), tool: "orix-convert \(CLI.version)", machine: machine(),471            system: ProcessInfo.processInfo.operatingSystemVersionString, loadAverageStart: loadStart, loadAverageEnd: loadAverage(), jobs: jobs, heavyJobs: heavy,472            timeoutSeconds: timeout, path: io.environment["PATH"] ?? "", toolsEnvironment: ProcessInfo.processInfo.environment["ORIX_CONVERT_TOOLS"].map { value in repository.map { value.replacingOccurrences(of: $0.path, with: "<repo>") } ?? value },473            wallSeconds: Date().timeIntervalSince(started), filter: filterText.isEmpty ? nil : filterText)474        let report = SweepReport(environment: environment, registryFingerprint: fingerprint, totals: totals, byEngine: byEngine,475                                 fixtures: allFixtures, formatsWithoutFixture: Dictionary(uniqueKeysWithValues: missing.map { ($0.key.rawValue, $0.value) }),476                                 detectorNotes: detectorNotes, inspections: inspections, chains: chainReport, temporaryLeftovers: leftovers,477                                 results: results, chainResults: chainResults.sorted { $0.id < $1.id })478        let suffix = filterText.isEmpty ? "" : "-partial"479        let jsonURL = reportDirectory.appendingPathComponent("e2e-matrix\(suffix).json"), markdownURL = reportDirectory.appendingPathComponent("e2e-matrix\(suffix).md")480        try Data((try Shared.json(report) + "\n").utf8).write(to: jsonURL, options: .atomic)481        try Data(markdown(report, workDirectory: workDirectory).utf8).write(to: markdownURL, options: .atomic)482        io.out("End-to-end sweep: \(totals.total) path(s) — \(totals.pass) passed, \(totals.fail) failed, \(totals.crash) crashed, \(totals.timeout) timed out, \(totals.skip) skipped\(totals.cancelled > 0 ? ", \(totals.cancelled) cancelled" : "") — \(Shared.seconds(environment.wallSeconds))")483        if let chains = chainReport {484            io.out("Chained plans: \(chains.totalPairs) pair(s) exist; sample of \(chains.sampled): \(chains.totals.pass) passed, \(chains.totals.fail + chains.totals.timeout) failed, \(chains.totals.refusedContent) refused: content, \(chains.totals.skip) skipped.")485        }486        io.out("Temporary work folders left behind: \(leftovers.isEmpty ? "none" : leftovers.joined(separator: ", "))")487        io.out("Report: \(markdownURL.path)")488        if InterruptCenter.shared.isInterrupted { return .interrupted }489        let chainFailures = (chainReport?.totals.fail ?? 0) + (chainReport?.totals.timeout ?? 0)490        return totals.fail + totals.crash + totals.timeout + chainFailures > 0 || !leftovers.isEmpty ? .conversionFailed : .success491    }492493    /// Runs one path alone in a child process (`matrix --verify --only-id`). A child killed by a signal is a crash.494    static func runInChild(path: ConversionPath, workDirectory: URL, timeout: Double) async -> SweepResult {495        var result = SweepResult(id: path.id, kind: "path", source: path.source.rawValue, destination: path.destination.rawValue, operation: path.operation,496                                 engines: [path.engine], status: "fail", seconds: nil, error: nil, errorKind: nil, failedCheck: nil, detectedAs: nil,497                                 explicitSourceFormat: false, fixtureOrigin: nil, fixtureVariant: nil, primaryFixtureError: nil, primaryFixtureOrigin: nil,498                                 primaryErrorKind: nil, variantsTried: nil, crashedWith: nil, options: [:], checks: 0, warnings: [], outputs: 0, outputBytes: nil, note: nil, refusalReason: nil)499        guard let executable = ToolEnvironment.executableURL() else { result.error = "The executable could not be located for the child run."; return result }500        let process = Process()501        process.executableURL = executable502        process.arguments = ["matrix", "--verify", "--work-dir", workDirectory.path, "--report-dir", workDirectory.appendingPathComponent("child-report").path,503                             "--only-id", path.id, "--timeout", String(timeout), "--no-chains"]504        let pipe = Pipe()505        process.standardOutput = pipe506        process.standardError = FileHandle.nullDevice507        process.standardInput = FileHandle.nullDevice508        let started = Date()509        let outcome: (Data, Process.TerminationReason, Int32)? = await withCheckedContinuation { continuation in510            DispatchQueue.global().async {511                do {512                    try process.run()513                    let data = pipe.fileHandleForReading.readDataToEndOfFile()514                    process.waitUntilExit()515                    continuation.resume(returning: (data, process.terminationReason, process.terminationStatus))516                } catch {517                    continuation.resume(returning: nil)518                }519            }520        }521        guard let (data, reason, status) = outcome else { result.error = "The child process could not be started."; return result }522        if reason == .uncaughtSignal {523            result.status = "crash"524            result.seconds = Date().timeIntervalSince(started)525            result.error = "The process was killed by signal \(status) while converting this path alone."526            result.errorKind = "crash"527            return result528        }529        if let decoded = try? JSONDecoder().decode(SweepResult.self, from: data) { return decoded }530        result.error = "The child process ended with status \(status) without a result."531        return result532    }533534    static func add(_ result: SweepResult, to totals: inout SweepReport.Totals) {535        totals.total += 1536        switch result.status {537        case "pass":538            totals.pass += 1539            if (result.fixtureVariant ?? 0) > 0 { totals.passOnVariantOnly += 1 }540        case "fail": totals.fail += 1541        case "crash": totals.crash += 1542        case "skip": totals.skip += 1543        case "timeout": totals.timeout += 1544        case "refused": totals.refusedContent += 1545        default: totals.cancelled += 1546        }547    }548549    static func markdown(_ report: SweepReport, workDirectory: URL) -> String {550        func cell(_ text: String) -> String { text.replacingOccurrences(of: "|", with: "\\|").replacingOccurrences(of: "\n", with: " ") }551        func seconds(_ value: Double?) -> String { value.map { String(format: "%.2f", locale: Locale(identifier: "en_US_POSIX"), $0) } ?? "—" }552        let e = report.environment, t = report.totals553        var out = "# ORIX Convert — end-to-end sweep of the conversion matrix\n\n"554        out += "> Generated file — do not edit. Written by `orix-convert matrix --verify` (`Scripts/matrix.sh --verify`).\n\n"555        out += "Every declared path was run as a user would run it: the file goes through the detector, the planner, the queue, the engine, the generic\n"556        out += "validation, the engine's own validation and the safe writer, into a temporary folder. Fixtures are real files: \(FixtureStore.seedFormats.count) seeds generated in code\n"557        out += "(\(FixtureStore.seedFormats.map(\.rawValue).joined(separator: ", "))); every other format was obtained by converting a fixture through the engines.\n"558        out += "Each path runs with its default settings and is forced onto the engine that declares it (so a path is tested even when the planner\n"559        out += "would prefer another engine for the same pair).\n\n"560        out += "| | |\n|---|---|\n| Date | \(e.date) |\n| Tool | \(e.tool) |\n| Machine | \(e.machine) |\n| System | \(e.system) |\n"561        out += "| Load average (1, 5, 15 min) at start | \(e.loadAverageStart.map { String($0) }.joined(separator: ", ")) |\n| Load average at end | \(e.loadAverageEnd.map { String($0) }.joined(separator: ", ")) |\n"562        out += "| Concurrency | \(e.jobs) jobs, \(e.heavyJobs) heavy |\n| Time limit per path | \(Int(e.timeoutSeconds)) s from job start |\n| PATH of the run | `\(e.path)` |\n"563        out += "| ORIX_CONVERT_TOOLS | \(e.toolsEnvironment.map { "`\($0)`" } ?? "not set") |\n| Wall time of this invocation | \(seconds(e.wallSeconds)) s (resumed runs only count the last invocation) |\n"564        out += "| Registry fingerprint | `\(report.registryFingerprint)` |\n"565        if let filter = e.filter { out += "| Filter | \(filter) — PARTIAL RUN |\n" }566        out += "\nSettings given to operations whose defaults ask for nothing (every other path ran with its defaults): "567        out += operationOptions.sorted { $0.key < $1.key }.map { op, values in "`\(op)` → " + values.sorted { $0.key < $1.key }.map { "`\($0.key)=\(Shared.optionValueText($0.value))`" }.joined(separator: " ") }.joined(separator: "; ") + ".\n"568        out += "\n## Totals\n\n| Paths | Passed | Failed | Crashed | Timed out | Skipped (no fixture) | Cancelled |\n|---:|---:|---:|---:|---:|---:|---:|\n"569        out += "| \(t.total) | \(t.pass) | \(t.fail) | \(t.crash) | \(t.timeout) | \(t.skip) | \(t.cancelled) |\n\n"570        out += "Of the \(t.pass) passes, \(t.passOnVariantOnly) needed an alternative fixture (listed below with the error met on the primary one).\n\n"571        out += "| Engine | Paths | Passed | Failed | Crashed | Timed out | Skipped |\n|---|---:|---:|---:|---:|---:|---:|\n"572        for (engine, c) in report.byEngine.sorted(by: { $0.key < $1.key }) {573            out += "| \(engine) | \(c.total) | \(c.pass) | \(c.fail) | \(c.crash) | \(c.timeout) | \(c.skip) |\n"574        }575        out += "\nTemporary work folders left behind after the run: \(report.temporaryLeftovers.isEmpty ? "none" : report.temporaryLeftovers.joined(separator: ", ")).\n\n"576577        let suspects = report.results.filter { $0.crashedWith != nil }578        if !suspects.isEmpty {579            out += "## Process crashes (\(suspects.count) path(s) involved)\n\n"580            out += "An invocation of the sweep was killed while these paths were being converted at the same time. Each was then re-run alone in a child process;\n"581            out += "the result of that re-run is the one counted.\n\n| Path | Re-run alone | Running at the same time when the process died |\n|---|---|---|\n"582            for r in suspects { out += "| `\(r.id)` | \(r.status)\(r.error.map { " — " + cell($0) } ?? "") | \((r.crashedWith ?? []).map { "`\($0)`" }.joined(separator: ", ")) |\n" }583            out += "\n"584        }585586        let failures = report.results.filter { $0.status == "fail" || $0.status == "timeout" || $0.status == "crash" }587        out += "## Failures (\(failures.count))\n\n"588        if failures.isEmpty {589            out += "None.\n\n"590        } else {591            out += "| Path | Result | Error kind | Failed check | Seconds | Error |\n|---|---|---|---|---:|---|\n"592            for f in failures {593                out += "| `\(f.id)` | \(f.status) | \(f.errorKind ?? "—") | \(f.failedCheck ?? "—") | \(seconds(f.seconds)) | \(cell(f.error ?? "")) (fixture: `\(f.fixtureOrigin ?? "?")`, \(f.variantsTried ?? 1) fixture(s) tried) |\n"594            }595            out += "\n"596        }597598        let variants = report.results.filter { $0.status == "pass" && ($0.fixtureVariant ?? 0) > 0 }599        out += "## Passed only on an alternative fixture (\(variants.count))\n\n"600        if variants.isEmpty {601            out += "None.\n\n"602        } else {603            out += "These paths failed on the primary fixture of their source format and passed on another real file of that format. They are counted as passed;\n"604            out += "the first error is kept here because it is either a legitimate refusal of that particular content or an integration defect.\n\n"605            out += "| Path | Primary fixture | Error on the primary | Passed with |\n|---|---|---|---|\n"606            for v in variants {607                out += "| `\(v.id)` | `\(v.primaryFixtureOrigin ?? "?")` | \(v.primaryErrorKind ?? "—"): \(cell(v.primaryFixtureError ?? "")) | `\(v.fixtureOrigin ?? "?")` |\n"608            }609            out += "\n"610        }611612        let skipped = report.results.filter { $0.status == "skip" }613        out += "## Skipped paths (\(skipped.count))\n\n"614        if skipped.isEmpty {615            out += "None: a fixture exists for every source format.\n\n"616        } else {617            out += "A path is skipped only when no real file of its source format could be produced.\n\n| Source format | Paths skipped | Why no fixture |\n|---|---:|---|\n"618            for (format, list) in Dictionary(grouping: skipped, by: \.source).sorted(by: { $0.key < $1.key }) {619                out += "| \(format) | \(list.count) | \(cell(report.formatsWithoutFixture[format] ?? "—")) |\n"620            }621            out += "\n"622        }623624        out += "## Detector and engine outputs\n\n"625        if report.detectorNotes.isEmpty {626            out += "The detector names every fixture as the format it is meant to be.\n\n"627        } else {628            out += "Fixtures the detector does not name as the format they stand for. Their paths were run with the source format given\n"629            out += "explicitly (a user dropping such a file would get the detected format's conversions instead).\n\n"630            out += "| Fixture of | Detected as | Confidence | Evidence | Listed as equivalent | Fixture made by |\n|---|---|---|---|---|---|\n"631            for n in report.detectorNotes {632                out += "| \(n.format) | \(n.detectedAs ?? "unrecognised") | \(n.confidence) | \(cell(n.evidence)) | \(n.acceptedAsEquivalent ? "yes" : "no") | `\(n.fixtureOrigin)` |\n"633            }634            out += "\n"635        }636637        out += "## Inspection\n\n`inspect` was run on the primary fixture of every format (the engine is the one the registry designates).\n\n"638        out += "| Inspected by | Formats |\n|---|---|\n"639        for (engine, list) in Dictionary(grouping: report.inspections.filter { $0.error == nil }, by: { $0.engine ?? "no engine (generic file facts only)" }).sorted(by: { $0.key < $1.key }) {640            out += "| \(engine) | \(list.map(\.format).joined(separator: ", ")) |\n"641        }642        let inspectionErrors = report.inspections.filter { $0.error != nil }643        if inspectionErrors.isEmpty {644            out += "\nNo inspection failed.\n\n"645        } else {646            out += "\n| Format | Inspection error |\n|---|---|\n"647            for note in inspectionErrors { out += "| \(note.format) | \(cell(note.error ?? "")) |\n" }648            out += "\n"649        }650651        if let chains = report.chains {652            out += "## Chained plans\n\n"653            out += "Pairs with no direct path that the planner bridges with two steps: **\(chains.totalPairs)**. A sample of \(chains.sampled) was run end to end "654            out += "(spread over the engine combinations): \(chains.totals.pass) passed (\(chains.totals.passOnVariantOnly) of them on an alternative fixture), \(chains.totals.fail) failed, \(chains.totals.timeout) timed out, "655            out += "\(chains.totals.refusedContent) refused: content, \(chains.totals.skip) skipped.\n\n"656            out += "A chain is first run on the primary fixture of its source format, then on the other real files of that format (for instance an SVG with\n"657            out += "text elements, a NetCDF grid with `lat` / `lon` coordinate variables). \"refused: content\" means that no fixture could satisfy the second step and\n"658            out += "that the engine explained the refusal by the content of the file (\(ContentRefusal.rules.map(\.reason).joined(separator: ", "))): it is reported, not counted as a failure.\n"659            out += "Any other error is a failure.\n\n"660            out += "| Engines (step 1 + step 2) | Pairs |\n|---|---:|\n"661            for (key, count) in chains.byEngines.sorted(by: { $0.key < $1.key }) { out += "| \(key) | \(count) |\n" }662            out += "\n| Chain run | Result | Fixture | Seconds | Checks | Error |\n|---|---|---|---:|---:|---|\n"663            for r in report.chainResults {664                let status = r.status == "refused" ? "refused: content (\(r.refusalReason ?? "?"))" : r.status665                let fixture = (r.fixtureVariant ?? 0) > 0 ? "`\(r.fixtureOrigin ?? "?")` (the primary fixture was refused: \(cell(r.primaryFixtureError ?? "")))" : "primary"666                out += "| \(r.note ?? r.id) (\(r.engines.joined(separator: " + "))) | \(status) | \(fixture) | \(seconds(r.seconds)) | \(r.checks) | \(cell(r.error ?? "")) |\n"667            }668            out += "\nThe complete list of chained pairs is in `e2e-matrix.json` (`chains.pairs`).\n\n"669        }670671        let slow = report.results.filter { $0.seconds != nil }.sorted { ($0.seconds ?? 0) > ($1.seconds ?? 0) }.prefix(15)672        out += "## Slowest paths\n\n| Path | Seconds | Result |\n|---|---:|---|\n"673        for r in slow { out += "| `\(r.id)` | \(seconds(r.seconds)) | \(r.status) |\n" }674        out += "\nTimes are job durations on a shared machine under the load shown above; they are not benchmarks.\n\n"675676        out += "## Fixtures\n\n| Format | Origin |\n|---|---|\n"677        for f in report.fixtures { out += "| \(f.format) | \(f.origin == "seed" ? "seed (generated in code)" : "`\(f.origin)`") |\n" }678        out += "\n## Every path\n\n| Path | Result | Seconds | Checks | Warnings | Outputs |\n|---|---|---:|---:|---|---:|\n"679        for r in report.results {680            out += "| `\(r.id)` | \(r.status)\(r.explicitSourceFormat ? " (format given)" : "") | \(seconds(r.seconds)) | \(r.checks) | \(r.warnings.isEmpty ? "—" : cell(Array(Set(r.warnings)).sorted().joined(separator: ", "))) | \(r.outputs) |\n"681        }682        return out683    }684}685
Files main
Menu

Theme