Repository Public License
1// ORIX Convert — Packages/EngineScientific/Sources/EngineScientificBench/main.swift2// Description: Large-dataset benchmark of the scientific engine. `generate` writes a multi-gigabyte HDF5 file through3// the engine's own NetCDF classic → NetCDF-4 path fed by a synthetic source; `convert` runs one4// conversion through the public engine API and prints elapsed time, throughput and peak resident5// memory (getrusage). Each phase is run as its own process so that peaks are not mixed.6// Author: Simon-Pierre Boucher — contact@spboucher.ai7import Foundation8import ConvertCore9import EngineScientific1011func peakRSS() -> Int64 {12 var usage = rusage()13 getrusage(RUSAGE_SELF, &usage)14 return Int64(usage.ru_maxrss) // bytes on macOS15}1617func fileSize(_ url: URL) -> Int64 {18 var total: Int64 = 019 var isDirectory: ObjCBool = false20 guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory) else { return 0 }21 if !isDirectory.boolValue { return (try? FileManager.default.attributesOfItem(atPath: url.path)[.size] as? NSNumber)?.int64Value ?? 0 }22 for case let item as URL in FileManager.default.enumerator(at: url, includingPropertiesForKeys: nil) ?? FileManager.default.enumerator(atPath: "/dev/null")! {23 total += (try? FileManager.default.attributesOfItem(atPath: item.path)[.size] as? NSNumber)?.int64Value ?? 024 }25 return total26}2728let args = CommandLine.arguments29guard args.count >= 2 else {30 print("""31 usage:32 EngineScientificBench generate-classic <out.nc> <elements> float64 /signal(n) in a CDF-5 file33 EngineScientificBench generate-grid <out.nc> <rows> <columns> int16 /grid(row, column) in a CDF-5 file34 EngineScientificBench convert <source> <hdf5|netcdf> <csv|parquet|json|hdf5|netcdf> <work-dir> [key=value …]35 """)36 exit(64)37}3839switch args[1] {40case "generate-classic", "generate-grid":41 // Written with plain file I/O following the NetCDF classic layout (CDF-5), so that generating the test data42 // does not exercise the code being measured.43 let url = URL(fileURLWithPath: args[2])44 let grid = args[1] == "generate-grid"45 let dims: [(String, UInt64)] = grid ? [("row", UInt64(args[3])!), ("column", UInt64(args[4])!)] : [("n", UInt64(args[3])!)]46 let name = grid ? "grid" : "signal"47 let elementSize: UInt64 = grid ? 2 : 848 let count = dims.reduce(UInt64(1)) { $0 * $1.1 }49 var h = Data([0x43, 0x44, 0x46, 5])50 func u32(_ v: UInt32) { withUnsafeBytes(of: v.bigEndian) { h.append(contentsOf: $0) } }51 func u64(_ v: UInt64) { withUnsafeBytes(of: v.bigEndian) { h.append(contentsOf: $0) } }52 func text(_ s: String) { u64(UInt64(s.utf8.count)); h.append(contentsOf: Array(s.utf8)); while h.count % 4 != 0 { h.append(0) } }53 u64(0)54 u32(0x0A); u64(UInt64(dims.count))55 for d in dims { text(d.0); u64(d.1) }56 u32(0); u64(0)57 u32(0x0B); u64(1)58 text(name)59 u64(UInt64(dims.count))60 for i in 0..<dims.count { u64(UInt64(i)) }61 u32(0); u64(0)62 u32(grid ? 3 : 6)63 u64((count * elementSize + 3) / 4 * 4)64 let begin = UInt64(h.count + 8)65 u64(begin)66 FileManager.default.createFile(atPath: url.path, contents: nil)67 let handle = try FileHandle(forWritingTo: url)68 try handle.write(contentsOf: h)69 let start = Date()70 let block = 4_000_00071 var written: UInt64 = 072 var buffer = Data(count: block * Int(elementSize))73 while written < count {74 let n = Int(min(UInt64(block), count - written))75 buffer.withUnsafeMutableBytes { raw in76 if grid {77 let p = raw.bindMemory(to: UInt16.self)78 for i in 0..<n { let v = written + UInt64(i); p[i] = UInt16(truncatingIfNeeded: (v &* 2654435761) >> 20).bigEndian }79 } else {80 let p = raw.bindMemory(to: UInt64.self)81 for i in 0..<n { let v = Double(written + UInt64(i)); p[i] = (sin(v * 1e-4) * 1000 + v * 1e-6).bitPattern.bigEndian }82 }83 }84 try handle.write(contentsOf: buffer.prefix(n * Int(elementSize)))85 written += UInt64(n)86 }87 try handle.close()88 print(String(format: "generated %@: %llu elements, %.2f GiB in %.1f s", url.lastPathComponent, count, Double(fileSize(url)) / Double(1 << 30), Date().timeIntervalSince(start)))8990case "convert":91 guard args.count >= 6 else { print("convert needs <source> <source-format> <destination-format> <work-dir>"); exit(64) }92 let source = URL(fileURLWithPath: args[2])93 let from = FormatID(args[3]), to = FormatID(args[4])94 let work = URL(fileURLWithPath: args[5], isDirectory: true)95 try? FileManager.default.createDirectory(at: work, withIntermediateDirectories: true)96 var options: [String: OptionValue] = [:]97 var operation: String?98 for pair in args.dropFirst(6) {99 let parts = pair.split(separator: "=", maxSplits: 1).map(String.init)100 guard parts.count == 2 else { continue }101 if parts[0] == "operation" { operation = parts[1]; continue }102 options[parts[0]] = Int(parts[1]).map { .int($0) } ?? (parts[1] == "true" ? .bool(true) : parts[1] == "false" ? .bool(false) : .string(parts[1]))103 }104 let engine = ScientificEngine()105 let request = ConversionRequest(source: source, sourceFormat: from, destinationFormat: to, operation: operation, options: ConversionOptions(options))106 let context = ConversionContext(workDirectory: work, warn: { FileHandle.standardError.write(Data("warning [\($0.code)]: \($0.message)\n".utf8)) })107 let start = Date()108 let output = try await engine.convert(request, context: context)109 let converted = Date().timeIntervalSince(start)110 let convertPeak = peakRSS()111 let checks = try await engine.validate(output, request: request, source: nil)112 let validated = Date().timeIntervalSince(start) - converted113 let inBytes = fileSize(source), outBytes = output.files.reduce(Int64(0)) { $0 + fileSize($1) }114 let elements = Double(output.facts["sci.rows"] ?? output.facts["sci.elements"] ?? "0") ?? 0115 print(String(format: "%@ → %@ %@: source %.2f GiB, output %.2f GiB, convert %.1f s (%.0f MiB/s of source, %.1f M values/s), validate %.1f s, peak RSS %.0f MiB (after validation %.0f MiB)",116 from.rawValue, to.rawValue, args.dropFirst(6).joined(separator: " "), Double(inBytes) / Double(1 << 30), Double(outBytes) / Double(1 << 30), converted,117 Double(inBytes) / Double(1 << 20) / converted, elements / 1e6 / converted, validated, Double(convertPeak) / Double(1 << 20), Double(peakRSS()) / Double(1 << 20)))118 for c in checks { print(" \(c.passed ? "ok " : "FAIL") \(c.name): \(c.detail)") }119 if checks.contains(where: { !$0.passed }) { exit(1) }120121default:122 print("unknown command \(args[1])")123 exit(64)124}125