SPB Git

Repository Public License

main.swift 5.7 KB 111 lines swift

1// ORIX Convert — Packages/EngineArchive/Sources/EngineArchiveBench/main.swift2// Description: Benchmark of the archive engine: creates and extracts one archive of a generated ~1 GB tree and prints3//              throughput (MB of uncompressed data per second) and the process's peak memory. One format per run,4//              so that the peak belongs to that format.5//              Usage: EngineArchiveBench <format> <scratch-folder> [megabytes] [store|fast|normal|maximum]6// Author: Simon-Pierre Boucher — contact@spboucher.ai7import ConvertCore8import EngineArchive9import Foundation1011func peakRSS() -> Int {12    var usage = rusage()13    getrusage(RUSAGE_SELF, &usage)14    return Int(usage.ru_maxrss)          // bytes on macOS15}1617/// A tree of mixed content: log-like text (very compressible), structured records (moderately compressible),18/// random bytes (incompressible) and many small files.19func generateTree(at root: URL, megabytes: Int) throws -> Int64 {20    let fm = FileManager.default21    try fm.createDirectory(at: root, withIntermediateDirectories: true)22    var total: Int64 = 023    func write(_ relative: String, _ data: Data) throws {24        let url = root.appendingPathComponent(relative)25        try fm.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)26        try data.write(to: url)27        total += Int64(data.count)28    }29    let unit = megabytes << 2030    // 40 % text.31    var line = 032    for file in 0..<8 {33        var data = Data(capacity: unit / 20)34        while data.count < unit / 20 {35            data.append(Data("2026-11-21T10:\(line % 60):\(line % 59) host-\(line % 13) INFO request \(line) path=/api/v1/items/\(line % 977) status=\(200 + line % 5) duration=\(line % 331)ms\n".utf8))36            line += 137        }38        try write("logs/server-\(file).log", data)39    }40    // 30 % records: random 8-byte keys with repetitive structure around them.41    for file in 0..<6 {42        var data = Data(capacity: unit / 20)43        var key = [UInt8](repeating: 0, count: 6)44        while data.count < unit / 20 {45            arc4random_buf(&key, key.count)46            data.append(Data("{\"id\":\"".utf8)); data.append(Data(key.map { String(format: "%02x", $0) }.joined().utf8)); data.append(Data("\",\"type\":\"measurement\",\"unit\":\"celsius\",\"ok\":true}\n".utf8))47        }48        try write("data/records-\(file).jsonl", data)49    }50    // 30 % random.51    for file in 0..<3 {52        var data = Data(count: unit / 10)53        data.withUnsafeMutableBytes { arc4random_buf($0.baseAddress, $0.count) }54        try write("media/blob-\(file).bin", data)55    }56    // 2 000 small files.57    for i in 0..<2000 { try write("src/module-\(i % 40)/file-\(i).txt", Data(String(repeating: "small file \(i) line\n", count: 200).utf8)) }58    return total59}6061func treeSize(_ root: URL) -> Int64 {62    var total: Int64 = 063    guard let enumerator = FileManager.default.enumerator(at: root, includingPropertiesForKeys: [.fileSizeKey, .isRegularFileKey]) else { return 0 }64    for case let url as URL in enumerator {65        let values = try? url.resourceValues(forKeys: [.fileSizeKey, .isRegularFileKey])66        if values?.isRegularFile == true { total += Int64(values?.fileSize ?? 0) }67    }68    return total69}7071let arguments = CommandLine.arguments72guard arguments.count >= 3, let format = FormatCatalog.format(FormatID(arguments[1]))?.id else {73    print("usage: EngineArchiveBench <zip|7z|tzst|txz|tgz|tbz2|tar> <scratch-folder> [megabytes]")74    exit(2)75}76let scratch = URL(fileURLWithPath: arguments[2], isDirectory: true)77let megabytes = arguments.count > 3 ? Int(arguments[3]) ?? 1024 : 102478let level = arguments.count > 4 ? arguments[4] : "normal"79let tree = scratch.appendingPathComponent("tree-\(megabytes)")80var bytes: Int64 = 081if FileManager.default.fileExists(atPath: tree.path) {82    bytes = treeSize(tree)83} else {84    bytes = try generateTree(at: tree, megabytes: megabytes)85}86let engine = ArchiveEngine()87let work = scratch.appendingPathComponent("work-\(format.rawValue)-\(getpid())")88try FileManager.default.createDirectory(at: work, withIntermediateDirectories: true)89defer { try? FileManager.default.removeItem(at: work) }90let megabytesOfData = Double(bytes) / 1_048_5769192var start = Date()93let created = try await engine.convert(ConversionRequest(source: tree, sourceFormat: .folder, destinationFormat: format, operation: ArchiveEngine.createOperation,94                                                         options: ConversionOptions(["compression.level": .string(level)])),95                                       context: ConversionContext(workDirectory: work))96let createSeconds = Date().timeIntervalSince(start)97let createPeak = peakRSS()98let archive = created.files[0]99let archiveBytes = (try archive.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0100101start = Date()102let extracted = try await engine.convert(ConversionRequest(source: archive, sourceFormat: format, destinationFormat: .folder, operation: ArchiveEngine.extractOperation, outputBaseName: "out"),103                                         context: ConversionContext(workDirectory: work))104let extractSeconds = Date().timeIntervalSince(start)105let peak = peakRSS()106guard extracted.facts["bytes"] == "\(bytes)" else { print("extracted \(extracted.facts["bytes"] ?? "?") bytes, expected \(bytes)"); exit(1) }107108print(String(format: "BENCH format=%@ level=\(level) data=%.0fMB archive=%.1fMB ratio=%.2f create=%.2fs (%.0f MB/s) extract=%.2fs (%.0f MB/s) peakRSS(create)=%.0fMB peakRSS(process)=%.0fMB",109             format.rawValue, megabytesOfData, Double(archiveBytes) / 1_048_576, Double(bytes) / Double(max(archiveBytes, 1)), createSeconds, megabytesOfData / createSeconds,110             extractSeconds, megabytesOfData / extractSeconds, Double(createPeak) / 1_048_576, Double(peak) / 1_048_576))111
Files main
Menu

Theme