Repository Public License
1// ORIX Convert — Packages/EngineData/Tests/EngineDataTests/Benchmarks.swift2// Description: Benchmarks recorded in docs/compatibility/engines/data.md. Disabled unless ORIX_DATA_BENCH names one;3// each benchmark runs in its own process so that the peak resident size is its own.4// Run: ORIX_DATA_BENCH=<name> ORIX_DATA_BENCH_DIR=<dir> swift test -c release -Xswiftc -enable-testing --filter Benchmarks5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import Testing8import ConvertCore9@testable import EngineData1011private let benchName = ProcessInfo.processInfo.environment["ORIX_DATA_BENCH"] ?? ""12private let benchDir = URL(fileURLWithPath: ProcessInfo.processInfo.environment["ORIX_DATA_BENCH_DIR"] ?? "/tmp/orix-data-bench", isDirectory: true)1314private func peakRSSMegabytes() -> Double {15 var usage = rusage()16 getrusage(RUSAGE_SELF, &usage)17 return Double(usage.ru_maxrss) / 1_048_576 // bytes on macOS18}1920private func fileSize(_ url: URL) -> Int { ((try? FileManager.default.attributesOfItem(atPath: url.path)[.size]) as? Int) ?? 0 }2122@Suite("Benchmarks", .enabled(if: !benchName.isEmpty, "set ORIX_DATA_BENCH to run"), .serialized)23struct Benchmarks {24 /// Generates the inputs with DuckDB: a wide-ish CSV of `ORIX_DATA_BENCH_ROWS` rows (default 22 M ≈ 2.2 GB),25 /// a 1 M-row CSV and a JSON Lines file.26 @Test(.enabled(if: benchName == "generate"))27 func generate() throws {28 try FileManager.default.createDirectory(at: benchDir, withIntermediateDirectories: true)29 let rows = Int(ProcessInfo.processInfo.environment["ORIX_DATA_BENCH_ROWS"] ?? "") ?? 22_000_00030 let ws = try Workspace(parent: benchDir)31 var o = DataOptions(); o.memoryLimitMB = 4096; o.threads = 832 let session = try Pipeline.makeSession(workspace: ws, sources: [], extraDirectories: [benchDir], options: o, control: nil)33 func select(_ n: Int) -> String {34 """35 SELECT range AS id, md5(CAST(range AS VARCHAR)) AS token, 'customer ' || (range % 100000) AS customer, round((range * 7919 % 1000000) / 100.0, 2) AS amount,36 range % 3 = 0 AS active, DATE '2020-01-01' + CAST(range % 2000 AS INTEGER) AS day, TIMESTAMP '2020-01-01 00:00:00' + to_seconds(range * 37 % 100000000) AS at,37 ['north', 'south', 'east', 'west', 'centre'][1 + range % 5] AS region, lpad(CAST(range % 99999 AS VARCHAR), 5, '0') AS zip38 FROM range(\(n))39 """40 }41 for (name, n) in [("big.csv", rows), ("million.csv", 1_000_000)] {42 let start = Date()43 try session.execute("COPY (\(select(n))) TO \(try SQL.path(benchDir.appendingPathComponent(name))) (FORMAT csv, HEADER true)")44 print("BENCH generate \(name): \(n) rows, \(fileSize(benchDir.appendingPathComponent(name)) >> 20) MB, \(String(format: "%.1f", Date().timeIntervalSince(start))) s")45 }46 let start = Date()47 try session.execute("COPY (\(select(8_000_000))) TO \(try SQL.path(benchDir.appendingPathComponent("big.jsonl"))) (FORMAT json, ARRAY false)")48 print("BENCH generate big.jsonl: 8000000 rows, \(fileSize(benchDir.appendingPathComponent("big.jsonl")) >> 20) MB, \(String(format: "%.1f", Date().timeIntervalSince(start))) s")49 }5051 static let cases: [(name: String, source: String, from: FormatID, to: FormatID, options: [String: OptionValue])] = [52 ("csv-parquet-zstd", "big.csv", .csv, .parquet, ["parquet.compression": .string("zstd")]),53 ("csv-parquet-snappy", "big.csv", .csv, .parquet, ["parquet.compression": .string("snappy")]),54 ("csv-parquet-zstd-1gb", "big.csv", .csv, .parquet, ["parquet.compression": .string("zstd"), "memory.limit.mb": .int(1024)]),55 ("csv-parquet-zstd-512mb", "big.csv", .csv, .parquet, ["parquet.compression": .string("zstd"), "memory.limit.mb": .int(512), "threads": .int(4)]),56 ("parquet-csv-512mb", "out/csv-parquet-zstd.parquet", .parquet, .csv, ["memory.limit.mb": .int(512), "threads": .int(4)]),57 ("csv-parquet-zstd-notypes", "big.csv", .csv, .parquet, ["parquet.compression": .string("zstd"), "types.detect": .bool(false)]),58 ("parquet-csv", "out/csv-parquet-zstd.parquet", .parquet, .csv, [:]),59 ("csv-xlsx", "million.csv", .csv, .xlsx, [:]),60 ("csv-xlsx-inline", "million.csv", .csv, .xlsx, ["xlsx.strings": .string("inline")]),61 ("xlsx-csv", "out/csv-xlsx.xlsx", .xlsx, .csv, [:]),62 ("csv-ods", "million.csv", .csv, .ods, [:]),63 ("jsonl-parquet", "big.jsonl", .jsonl, .parquet, [:]),64 ("csv-sqlite", "big.csv", .csv, .sqlite, [:]),65 ("csv-arrow", "big.csv", .csv, .arrow, [:]),66 ]6768 @Test(.enabled(if: Benchmarks.cases.contains { $0.name == benchName }))69 func run() async throws {70 let c = try #require(Benchmarks.cases.first { $0.name == benchName })71 let source = benchDir.appendingPathComponent(c.source)72 let work = benchDir.appendingPathComponent("work-\(c.name)", isDirectory: true)73 try? FileManager.default.removeItem(at: work)74 try FileManager.default.createDirectory(at: work, withIntermediateDirectories: true)75 let request = ConversionRequest(source: source, sourceFormat: c.from, destinationFormat: c.to, options: .init(c.options), outputBaseName: c.name)76 let bag = WarningBag()77 let engine = DataEngine()78 let start = Date()79 let output = try await engine.convert(request, context: ConversionContext(workDirectory: work, progress: { bag.progress($0) }, warn: { bag.add($0) }))80 let seconds = Date().timeIntervalSince(start)81 let peak = peakRSSMegabytes()82 let rows = Double(output.facts["rows"] ?? "0") ?? 083 let outDir = benchDir.appendingPathComponent("out", isDirectory: true)84 try FileManager.default.createDirectory(at: outDir, withIntermediateDirectories: true)85 let kept = outDir.appendingPathComponent(output.files[0].lastPathComponent)86 try? FileManager.default.removeItem(at: kept)87 try FileManager.default.moveItem(at: output.files[0], to: kept)88 let fractions = bag.progressValues.compactMap { $0 }89 print("BENCH \(c.name): source \(fileSize(source) >> 20) MB, \(Int(rows)) rows, \(String(format: "%.2f", seconds)) s, \(Int(rows / seconds)) rows/s, \(String(format: "%.0f", Double(fileSize(source)) / 1_048_576 / seconds)) MB/s, output \(fileSize(kept) >> 20) MB, peak RSS \(Int(peak)) MB, progress updates \(fractions.count), warnings \(bag.codes.sorted())")90 if ProcessInfo.processInfo.environment["ORIX_DATA_BENCH_VALIDATE"] == "1" {91 let v = Date()92 let checks = try await engine.validate(ConversionOutput(file: kept, facts: output.facts), request: request, source: nil)93 print("BENCH \(c.name) validate: \(String(format: "%.2f", Date().timeIntervalSince(v))) s, \(checks.map { "\($0.name)=\($0.passed)" }.joined(separator: " "))")94 }95 try? FileManager.default.removeItem(at: work)96 }9798 /// Writes the path table of docs/compatibility/engines/data.md from the engine's own declarations.99 @Test(.enabled(if: benchName == "paths"))100 func pathTable() throws {101 func reader(_ f: FormatID) -> String {102 switch f {103 case .csv, .tsv: "DuckDB `read_csv` + engine type detection"104 case .jsonl: "DuckDB `read_json`"105 case .json: "DuckDB `read_json` (top-level array) / engine JSON parser (object)"106 case .parquet: "DuckDB `read_parquet`"107 case .duckdb: "DuckDB `ATTACH` (read-only)"108 case .sqlite: "system libsqlite3 → staging"109 case .xlsx: "engine XLSX reader → staging"110 case .ods: "engine ODS reader → staging"111 case .xls: "LibreOffice → XLSX → engine XLSX reader"112 case .arrow: "engine Arrow IPC reader → staging"113 case .xml: "XMLParser → records"114 case .yaml: "Yams → records"115 case .toml: "engine TOML parser → records"116 case .plist: "PropertyListSerialization → records"117 default: "?"118 }119 }120 func writer(_ f: FormatID) -> String {121 switch f {122 case .csv, .tsv: "DuckDB `COPY … (FORMAT csv)`"123 case .json, .jsonl: "DuckDB `COPY … (FORMAT json)`"124 case .parquet: "DuckDB `COPY … (FORMAT parquet)`"125 case .duckdb: "DuckDB `CREATE TABLE AS`"126 case .sqlite: "system libsqlite3"127 case .xlsx: "engine XLSX writer"128 case .ods: "engine ODS writer"129 case .xls: "engine XLSX writer → LibreOffice"130 case .arrow: "engine Arrow IPC writer"131 case .xml: "engine XML writer"132 case .yaml: "Yams (row by row)"133 case .toml: "engine TOML writer"134 case .plist: "PropertyListSerialization"135 default: "?"136 }137 }138 func tree(_ f: FormatID) -> String {139 switch f {140 case .json: "engine JSON codec"141 case .yaml: "Yams"142 case .toml: "engine TOML codec"143 case .xml: "XMLParser / engine XML writer"144 default: "PropertyListSerialization"145 }146 }147 var out = "| Source | Destination | Operation | Status | Fidelity | Preserves | Backend (read → write) | Test |\n|---|---|---|---|---|---|---|---|\n"148 var limits: [String: [String]] = [:]149 var order: [String] = []150 for p in DataEngine().paths {151 let isTree = FormatClass.isTreePath(p.source, p.destination)152 let backend = isTree ? "\(tree(p.source)) → \(tree(p.destination))" : "\(reader(p.source)) → \(writer(p.destination))"153 let test = isTree ? "`TreeTests.path` (\(p.source) → \(p.destination))" : "`MatrixTests.path` (\(p.source) → \(p.destination))"154 out += "| \(p.source.rawValue) | \(p.destination.rawValue) | — | \(p.status.rawValue) | \(p.fidelity.rawValue) | \(p.preserves.map(\.rawValue).sorted().joined(separator: ", ")) | \(backend) | \(test) |\n"155 for l in p.limitations {156 if limits[l] == nil { order.append(l) }157 limits[l, default: []].append("\(p.source.rawValue)→\(p.destination.rawValue)")158 }159 }160 var lim = ""161 for l in order {162 let paths = limits[l] ?? []163 let sources = Set(paths.map { $0.split(separator: "→")[0] }), dests = Set(paths.map { $0.split(separator: "→")[1] })164 let scope = paths.count <= 6 ? paths.joined(separator: ", ") : (sources.count < dests.count ? "from \(sources.sorted().joined(separator: ", ")) (\(paths.count) paths)" : "to \(dests.sorted().joined(separator: ", ")) (\(paths.count) paths)")165 lim += "- \(l) — *\(scope)*\n"166 }167 try out.write(to: benchDir.appendingPathComponent("paths.md"), atomically: true, encoding: .utf8)168 try lim.write(to: benchDir.appendingPathComponent("limitations.md"), atomically: true, encoding: .utf8)169 print("BENCH paths: \(DataEngine().paths.count)")170 }171}172