SPB Git

Repository Public License

main.swift 6.7 KB 125 lines swift

1// ORIX Convert — Packages/EngineScientific/Sources/EngineScientificBench/main.swift2// Description: Large-dataset benchmark of the scientific engine. `generate` writes a multi-gigabyte HDF5 file through3//              the engine's own NetCDF classic → NetCDF-4 path fed by a synthetic source; `convert` runs one4//              conversion through the public engine API and prints elapsed time, throughput and peak resident5//              memory (getrusage). Each phase is run as its own process so that peaks are not mixed.6// Author: Simon-Pierre Boucher — contact@spboucher.ai7import Foundation8import ConvertCore9import EngineScientific1011func peakRSS() -> Int64 {12    var usage = rusage()13    getrusage(RUSAGE_SELF, &usage)14    return Int64(usage.ru_maxrss)   // bytes on macOS15}1617func fileSize(_ url: URL) -> Int64 {18    var total: Int64 = 019    var isDirectory: ObjCBool = false20    guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory) else { return 0 }21    if !isDirectory.boolValue { return (try? FileManager.default.attributesOfItem(atPath: url.path)[.size] as? NSNumber)?.int64Value ?? 0 }22    for case let item as URL in FileManager.default.enumerator(at: url, includingPropertiesForKeys: nil) ?? FileManager.default.enumerator(atPath: "/dev/null")! {23        total += (try? FileManager.default.attributesOfItem(atPath: item.path)[.size] as? NSNumber)?.int64Value ?? 024    }25    return total26}2728let args = CommandLine.arguments29guard args.count >= 2 else {30    print("""31    usage:32      EngineScientificBench generate-classic <out.nc> <elements>            float64 /signal(n) in a CDF-5 file33      EngineScientificBench generate-grid <out.nc> <rows> <columns>         int16 /grid(row, column) in a CDF-5 file34      EngineScientificBench convert <source> <hdf5|netcdf> <csv|parquet|json|hdf5|netcdf> <work-dir> [key=value …]35    """)36    exit(64)37}3839switch args[1] {40case "generate-classic", "generate-grid":41    // Written with plain file I/O following the NetCDF classic layout (CDF-5), so that generating the test data42    // does not exercise the code being measured.43    let url = URL(fileURLWithPath: args[2])44    let grid = args[1] == "generate-grid"45    let dims: [(String, UInt64)] = grid ? [("row", UInt64(args[3])!), ("column", UInt64(args[4])!)] : [("n", UInt64(args[3])!)]46    let name = grid ? "grid" : "signal"47    let elementSize: UInt64 = grid ? 2 : 848    let count = dims.reduce(UInt64(1)) { $0 * $1.1 }49    var h = Data([0x43, 0x44, 0x46, 5])50    func u32(_ v: UInt32) { withUnsafeBytes(of: v.bigEndian) { h.append(contentsOf: $0) } }51    func u64(_ v: UInt64) { withUnsafeBytes(of: v.bigEndian) { h.append(contentsOf: $0) } }52    func text(_ s: String) { u64(UInt64(s.utf8.count)); h.append(contentsOf: Array(s.utf8)); while h.count % 4 != 0 { h.append(0) } }53    u64(0)54    u32(0x0A); u64(UInt64(dims.count))55    for d in dims { text(d.0); u64(d.1) }56    u32(0); u64(0)57    u32(0x0B); u64(1)58    text(name)59    u64(UInt64(dims.count))60    for i in 0..<dims.count { u64(UInt64(i)) }61    u32(0); u64(0)62    u32(grid ? 3 : 6)63    u64((count * elementSize + 3) / 4 * 4)64    let begin = UInt64(h.count + 8)65    u64(begin)66    FileManager.default.createFile(atPath: url.path, contents: nil)67    let handle = try FileHandle(forWritingTo: url)68    try handle.write(contentsOf: h)69    let start = Date()70    let block = 4_000_00071    var written: UInt64 = 072    var buffer = Data(count: block * Int(elementSize))73    while written < count {74        let n = Int(min(UInt64(block), count - written))75        buffer.withUnsafeMutableBytes { raw in76            if grid {77                let p = raw.bindMemory(to: UInt16.self)78                for i in 0..<n { let v = written + UInt64(i); p[i] = UInt16(truncatingIfNeeded: (v &* 2654435761) >> 20).bigEndian }79            } else {80                let p = raw.bindMemory(to: UInt64.self)81                for i in 0..<n { let v = Double(written + UInt64(i)); p[i] = (sin(v * 1e-4) * 1000 + v * 1e-6).bitPattern.bigEndian }82            }83        }84        try handle.write(contentsOf: buffer.prefix(n * Int(elementSize)))85        written += UInt64(n)86    }87    try handle.close()88    print(String(format: "generated %@: %llu elements, %.2f GiB in %.1f s", url.lastPathComponent, count, Double(fileSize(url)) / Double(1 << 30), Date().timeIntervalSince(start)))8990case "convert":91    guard args.count >= 6 else { print("convert needs <source> <source-format> <destination-format> <work-dir>"); exit(64) }92    let source = URL(fileURLWithPath: args[2])93    let from = FormatID(args[3]), to = FormatID(args[4])94    let work = URL(fileURLWithPath: args[5], isDirectory: true)95    try? FileManager.default.createDirectory(at: work, withIntermediateDirectories: true)96    var options: [String: OptionValue] = [:]97    var operation: String?98    for pair in args.dropFirst(6) {99        let parts = pair.split(separator: "=", maxSplits: 1).map(String.init)100        guard parts.count == 2 else { continue }101        if parts[0] == "operation" { operation = parts[1]; continue }102        options[parts[0]] = Int(parts[1]).map { .int($0) } ?? (parts[1] == "true" ? .bool(true) : parts[1] == "false" ? .bool(false) : .string(parts[1]))103    }104    let engine = ScientificEngine()105    let request = ConversionRequest(source: source, sourceFormat: from, destinationFormat: to, operation: operation, options: ConversionOptions(options))106    let context = ConversionContext(workDirectory: work, warn: { FileHandle.standardError.write(Data("warning [\($0.code)]: \($0.message)\n".utf8)) })107    let start = Date()108    let output = try await engine.convert(request, context: context)109    let converted = Date().timeIntervalSince(start)110    let convertPeak = peakRSS()111    let checks = try await engine.validate(output, request: request, source: nil)112    let validated = Date().timeIntervalSince(start) - converted113    let inBytes = fileSize(source), outBytes = output.files.reduce(Int64(0)) { $0 + fileSize($1) }114    let elements = Double(output.facts["sci.rows"] ?? output.facts["sci.elements"] ?? "0") ?? 0115    print(String(format: "%@ → %@ %@: source %.2f GiB, output %.2f GiB, convert %.1f s (%.0f MiB/s of source, %.1f M values/s), validate %.1f s, peak RSS %.0f MiB (after validation %.0f MiB)",116                 from.rawValue, to.rawValue, args.dropFirst(6).joined(separator: " "), Double(inBytes) / Double(1 << 30), Double(outBytes) / Double(1 << 30), converted,117                 Double(inBytes) / Double(1 << 20) / converted, elements / 1e6 / converted, validated, Double(convertPeak) / Double(1 << 20), Double(peakRSS()) / Double(1 << 20)))118    for c in checks { print("  \(c.passed ? "ok  " : "FAIL") \(c.name): \(c.detail)") }119    if checks.contains(where: { !$0.passed }) { exit(1) }120121default:122    print("unknown command \(args[1])")123    exit(64)124}125
Files main
Menu

Theme