SPB Git

Repository Public License

Benchmarks.swift 10.9 KB 172 lines swift

1// ORIX Convert — Packages/EngineData/Tests/EngineDataTests/Benchmarks.swift2// Description: Benchmarks recorded in docs/compatibility/engines/data.md. Disabled unless ORIX_DATA_BENCH names one;3//              each benchmark runs in its own process so that the peak resident size is its own.4//              Run: ORIX_DATA_BENCH=<name> ORIX_DATA_BENCH_DIR=<dir> swift test -c release -Xswiftc -enable-testing --filter Benchmarks5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import Testing8import ConvertCore9@testable import EngineData1011private let benchName = ProcessInfo.processInfo.environment["ORIX_DATA_BENCH"] ?? ""12private let benchDir = URL(fileURLWithPath: ProcessInfo.processInfo.environment["ORIX_DATA_BENCH_DIR"] ?? "/tmp/orix-data-bench", isDirectory: true)1314private func peakRSSMegabytes() -> Double {15    var usage = rusage()16    getrusage(RUSAGE_SELF, &usage)17    return Double(usage.ru_maxrss) / 1_048_576   // bytes on macOS18}1920private func fileSize(_ url: URL) -> Int { ((try? FileManager.default.attributesOfItem(atPath: url.path)[.size]) as? Int) ?? 0 }2122@Suite("Benchmarks", .enabled(if: !benchName.isEmpty, "set ORIX_DATA_BENCH to run"), .serialized)23struct Benchmarks {24    /// Generates the inputs with DuckDB: a wide-ish CSV of `ORIX_DATA_BENCH_ROWS` rows (default 22 M ≈ 2.2 GB),25    /// a 1 M-row CSV and a JSON Lines file.26    @Test(.enabled(if: benchName == "generate"))27    func generate() throws {28        try FileManager.default.createDirectory(at: benchDir, withIntermediateDirectories: true)29        let rows = Int(ProcessInfo.processInfo.environment["ORIX_DATA_BENCH_ROWS"] ?? "") ?? 22_000_00030        let ws = try Workspace(parent: benchDir)31        var o = DataOptions(); o.memoryLimitMB = 4096; o.threads = 832        let session = try Pipeline.makeSession(workspace: ws, sources: [], extraDirectories: [benchDir], options: o, control: nil)33        func select(_ n: Int) -> String {34            """35            SELECT range AS id, md5(CAST(range AS VARCHAR)) AS token, 'customer ' || (range % 100000) AS customer, round((range * 7919 % 1000000) / 100.0, 2) AS amount,36                   range % 3 = 0 AS active, DATE '2020-01-01' + CAST(range % 2000 AS INTEGER) AS day, TIMESTAMP '2020-01-01 00:00:00' + to_seconds(range * 37 % 100000000) AS at,37                   ['north', 'south', 'east', 'west', 'centre'][1 + range % 5] AS region, lpad(CAST(range % 99999 AS VARCHAR), 5, '0') AS zip38            FROM range(\(n))39            """40        }41        for (name, n) in [("big.csv", rows), ("million.csv", 1_000_000)] {42            let start = Date()43            try session.execute("COPY (\(select(n))) TO \(try SQL.path(benchDir.appendingPathComponent(name))) (FORMAT csv, HEADER true)")44            print("BENCH generate \(name): \(n) rows, \(fileSize(benchDir.appendingPathComponent(name)) >> 20) MB, \(String(format: "%.1f", Date().timeIntervalSince(start))) s")45        }46        let start = Date()47        try session.execute("COPY (\(select(8_000_000))) TO \(try SQL.path(benchDir.appendingPathComponent("big.jsonl"))) (FORMAT json, ARRAY false)")48        print("BENCH generate big.jsonl: 8000000 rows, \(fileSize(benchDir.appendingPathComponent("big.jsonl")) >> 20) MB, \(String(format: "%.1f", Date().timeIntervalSince(start))) s")49    }5051    static let cases: [(name: String, source: String, from: FormatID, to: FormatID, options: [String: OptionValue])] = [52        ("csv-parquet-zstd", "big.csv", .csv, .parquet, ["parquet.compression": .string("zstd")]),53        ("csv-parquet-snappy", "big.csv", .csv, .parquet, ["parquet.compression": .string("snappy")]),54        ("csv-parquet-zstd-1gb", "big.csv", .csv, .parquet, ["parquet.compression": .string("zstd"), "memory.limit.mb": .int(1024)]),55        ("csv-parquet-zstd-512mb", "big.csv", .csv, .parquet, ["parquet.compression": .string("zstd"), "memory.limit.mb": .int(512), "threads": .int(4)]),56        ("parquet-csv-512mb", "out/csv-parquet-zstd.parquet", .parquet, .csv, ["memory.limit.mb": .int(512), "threads": .int(4)]),57        ("csv-parquet-zstd-notypes", "big.csv", .csv, .parquet, ["parquet.compression": .string("zstd"), "types.detect": .bool(false)]),58        ("parquet-csv", "out/csv-parquet-zstd.parquet", .parquet, .csv, [:]),59        ("csv-xlsx", "million.csv", .csv, .xlsx, [:]),60        ("csv-xlsx-inline", "million.csv", .csv, .xlsx, ["xlsx.strings": .string("inline")]),61        ("xlsx-csv", "out/csv-xlsx.xlsx", .xlsx, .csv, [:]),62        ("csv-ods", "million.csv", .csv, .ods, [:]),63        ("jsonl-parquet", "big.jsonl", .jsonl, .parquet, [:]),64        ("csv-sqlite", "big.csv", .csv, .sqlite, [:]),65        ("csv-arrow", "big.csv", .csv, .arrow, [:]),66    ]6768    @Test(.enabled(if: Benchmarks.cases.contains { $0.name == benchName }))69    func run() async throws {70        let c = try #require(Benchmarks.cases.first { $0.name == benchName })71        let source = benchDir.appendingPathComponent(c.source)72        let work = benchDir.appendingPathComponent("work-\(c.name)", isDirectory: true)73        try? FileManager.default.removeItem(at: work)74        try FileManager.default.createDirectory(at: work, withIntermediateDirectories: true)75        let request = ConversionRequest(source: source, sourceFormat: c.from, destinationFormat: c.to, options: .init(c.options), outputBaseName: c.name)76        let bag = WarningBag()77        let engine = DataEngine()78        let start = Date()79        let output = try await engine.convert(request, context: ConversionContext(workDirectory: work, progress: { bag.progress($0) }, warn: { bag.add($0) }))80        let seconds = Date().timeIntervalSince(start)81        let peak = peakRSSMegabytes()82        let rows = Double(output.facts["rows"] ?? "0") ?? 083        let outDir = benchDir.appendingPathComponent("out", isDirectory: true)84        try FileManager.default.createDirectory(at: outDir, withIntermediateDirectories: true)85        let kept = outDir.appendingPathComponent(output.files[0].lastPathComponent)86        try? FileManager.default.removeItem(at: kept)87        try FileManager.default.moveItem(at: output.files[0], to: kept)88        let fractions = bag.progressValues.compactMap { $0 }89        print("BENCH \(c.name): source \(fileSize(source) >> 20) MB, \(Int(rows)) rows, \(String(format: "%.2f", seconds)) s, \(Int(rows / seconds)) rows/s, \(String(format: "%.0f", Double(fileSize(source)) / 1_048_576 / seconds)) MB/s, output \(fileSize(kept) >> 20) MB, peak RSS \(Int(peak)) MB, progress updates \(fractions.count), warnings \(bag.codes.sorted())")90        if ProcessInfo.processInfo.environment["ORIX_DATA_BENCH_VALIDATE"] == "1" {91            let v = Date()92            let checks = try await engine.validate(ConversionOutput(file: kept, facts: output.facts), request: request, source: nil)93            print("BENCH \(c.name) validate: \(String(format: "%.2f", Date().timeIntervalSince(v))) s, \(checks.map { "\($0.name)=\($0.passed)" }.joined(separator: " "))")94        }95        try? FileManager.default.removeItem(at: work)96    }9798    /// Writes the path table of docs/compatibility/engines/data.md from the engine's own declarations.99    @Test(.enabled(if: benchName == "paths"))100    func pathTable() throws {101        func reader(_ f: FormatID) -> String {102            switch f {103            case .csv, .tsv: "DuckDB `read_csv` + engine type detection"104            case .jsonl: "DuckDB `read_json`"105            case .json: "DuckDB `read_json` (top-level array) / engine JSON parser (object)"106            case .parquet: "DuckDB `read_parquet`"107            case .duckdb: "DuckDB `ATTACH` (read-only)"108            case .sqlite: "system libsqlite3 → staging"109            case .xlsx: "engine XLSX reader → staging"110            case .ods: "engine ODS reader → staging"111            case .xls: "LibreOffice → XLSX → engine XLSX reader"112            case .arrow: "engine Arrow IPC reader → staging"113            case .xml: "XMLParser → records"114            case .yaml: "Yams → records"115            case .toml: "engine TOML parser → records"116            case .plist: "PropertyListSerialization → records"117            default: "?"118            }119        }120        func writer(_ f: FormatID) -> String {121            switch f {122            case .csv, .tsv: "DuckDB `COPY … (FORMAT csv)`"123            case .json, .jsonl: "DuckDB `COPY … (FORMAT json)`"124            case .parquet: "DuckDB `COPY … (FORMAT parquet)`"125            case .duckdb: "DuckDB `CREATE TABLE AS`"126            case .sqlite: "system libsqlite3"127            case .xlsx: "engine XLSX writer"128            case .ods: "engine ODS writer"129            case .xls: "engine XLSX writer → LibreOffice"130            case .arrow: "engine Arrow IPC writer"131            case .xml: "engine XML writer"132            case .yaml: "Yams (row by row)"133            case .toml: "engine TOML writer"134            case .plist: "PropertyListSerialization"135            default: "?"136            }137        }138        func tree(_ f: FormatID) -> String {139            switch f {140            case .json: "engine JSON codec"141            case .yaml: "Yams"142            case .toml: "engine TOML codec"143            case .xml: "XMLParser / engine XML writer"144            default: "PropertyListSerialization"145            }146        }147        var out = "| Source | Destination | Operation | Status | Fidelity | Preserves | Backend (read → write) | Test |\n|---|---|---|---|---|---|---|---|\n"148        var limits: [String: [String]] = [:]149        var order: [String] = []150        for p in DataEngine().paths {151            let isTree = FormatClass.isTreePath(p.source, p.destination)152            let backend = isTree ? "\(tree(p.source)) → \(tree(p.destination))" : "\(reader(p.source)) → \(writer(p.destination))"153            let test = isTree ? "`TreeTests.path` (\(p.source) → \(p.destination))" : "`MatrixTests.path` (\(p.source) → \(p.destination))"154            out += "| \(p.source.rawValue) | \(p.destination.rawValue) | — | \(p.status.rawValue) | \(p.fidelity.rawValue) | \(p.preserves.map(\.rawValue).sorted().joined(separator: ", ")) | \(backend) | \(test) |\n"155            for l in p.limitations {156                if limits[l] == nil { order.append(l) }157                limits[l, default: []].append("\(p.source.rawValue)→\(p.destination.rawValue)")158            }159        }160        var lim = ""161        for l in order {162            let paths = limits[l] ?? []163            let sources = Set(paths.map { $0.split(separator: "→")[0] }), dests = Set(paths.map { $0.split(separator: "→")[1] })164            let scope = paths.count <= 6 ? paths.joined(separator: ", ") : (sources.count < dests.count ? "from \(sources.sorted().joined(separator: ", ")) (\(paths.count) paths)" : "to \(dests.sorted().joined(separator: ", ")) (\(paths.count) paths)")165            lim += "- \(l) — *\(scope)*\n"166        }167        try out.write(to: benchDir.appendingPathComponent("paths.md"), atomically: true, encoding: .utf8)168        try lim.write(to: benchDir.appendingPathComponent("limitations.md"), atomically: true, encoding: .utf8)169        print("BENCH paths: \(DataEngine().paths.count)")170    }171}172
Files main
Menu

Theme