Repository Public License
1// ORIX Convert — Packages/EngineArchive/Sources/EngineArchiveBench/main.swift2// Description: Benchmark of the archive engine: creates and extracts one archive of a generated ~1 GB tree and prints3// throughput (MB of uncompressed data per second) and the process's peak memory. One format per run,4// so that the peak belongs to that format.5// Usage: EngineArchiveBench <format> <scratch-folder> [megabytes] [store|fast|normal|maximum]6// Author: Simon-Pierre Boucher — contact@spboucher.ai7import ConvertCore8import EngineArchive9import Foundation1011func peakRSS() -> Int {12 var usage = rusage()13 getrusage(RUSAGE_SELF, &usage)14 return Int(usage.ru_maxrss) // bytes on macOS15}1617/// A tree of mixed content: log-like text (very compressible), structured records (moderately compressible),18/// random bytes (incompressible) and many small files.19func generateTree(at root: URL, megabytes: Int) throws -> Int64 {20 let fm = FileManager.default21 try fm.createDirectory(at: root, withIntermediateDirectories: true)22 var total: Int64 = 023 func write(_ relative: String, _ data: Data) throws {24 let url = root.appendingPathComponent(relative)25 try fm.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true)26 try data.write(to: url)27 total += Int64(data.count)28 }29 let unit = megabytes << 2030 // 40 % text.31 var line = 032 for file in 0..<8 {33 var data = Data(capacity: unit / 20)34 while data.count < unit / 20 {35 data.append(Data("2026-11-21T10:\(line % 60):\(line % 59) host-\(line % 13) INFO request \(line) path=/api/v1/items/\(line % 977) status=\(200 + line % 5) duration=\(line % 331)ms\n".utf8))36 line += 137 }38 try write("logs/server-\(file).log", data)39 }40 // 30 % records: random 8-byte keys with repetitive structure around them.41 for file in 0..<6 {42 var data = Data(capacity: unit / 20)43 var key = [UInt8](repeating: 0, count: 6)44 while data.count < unit / 20 {45 arc4random_buf(&key, key.count)46 data.append(Data("{\"id\":\"".utf8)); data.append(Data(key.map { String(format: "%02x", $0) }.joined().utf8)); data.append(Data("\",\"type\":\"measurement\",\"unit\":\"celsius\",\"ok\":true}\n".utf8))47 }48 try write("data/records-\(file).jsonl", data)49 }50 // 30 % random.51 for file in 0..<3 {52 var data = Data(count: unit / 10)53 data.withUnsafeMutableBytes { arc4random_buf($0.baseAddress, $0.count) }54 try write("media/blob-\(file).bin", data)55 }56 // 2 000 small files.57 for i in 0..<2000 { try write("src/module-\(i % 40)/file-\(i).txt", Data(String(repeating: "small file \(i) line\n", count: 200).utf8)) }58 return total59}6061func treeSize(_ root: URL) -> Int64 {62 var total: Int64 = 063 guard let enumerator = FileManager.default.enumerator(at: root, includingPropertiesForKeys: [.fileSizeKey, .isRegularFileKey]) else { return 0 }64 for case let url as URL in enumerator {65 let values = try? url.resourceValues(forKeys: [.fileSizeKey, .isRegularFileKey])66 if values?.isRegularFile == true { total += Int64(values?.fileSize ?? 0) }67 }68 return total69}7071let arguments = CommandLine.arguments72guard arguments.count >= 3, let format = FormatCatalog.format(FormatID(arguments[1]))?.id else {73 print("usage: EngineArchiveBench <zip|7z|tzst|txz|tgz|tbz2|tar> <scratch-folder> [megabytes]")74 exit(2)75}76let scratch = URL(fileURLWithPath: arguments[2], isDirectory: true)77let megabytes = arguments.count > 3 ? Int(arguments[3]) ?? 1024 : 102478let level = arguments.count > 4 ? arguments[4] : "normal"79let tree = scratch.appendingPathComponent("tree-\(megabytes)")80var bytes: Int64 = 081if FileManager.default.fileExists(atPath: tree.path) {82 bytes = treeSize(tree)83} else {84 bytes = try generateTree(at: tree, megabytes: megabytes)85}86let engine = ArchiveEngine()87let work = scratch.appendingPathComponent("work-\(format.rawValue)-\(getpid())")88try FileManager.default.createDirectory(at: work, withIntermediateDirectories: true)89defer { try? FileManager.default.removeItem(at: work) }90let megabytesOfData = Double(bytes) / 1_048_5769192var start = Date()93let created = try await engine.convert(ConversionRequest(source: tree, sourceFormat: .folder, destinationFormat: format, operation: ArchiveEngine.createOperation,94 options: ConversionOptions(["compression.level": .string(level)])),95 context: ConversionContext(workDirectory: work))96let createSeconds = Date().timeIntervalSince(start)97let createPeak = peakRSS()98let archive = created.files[0]99let archiveBytes = (try archive.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0100101start = Date()102let extracted = try await engine.convert(ConversionRequest(source: archive, sourceFormat: format, destinationFormat: .folder, operation: ArchiveEngine.extractOperation, outputBaseName: "out"),103 context: ConversionContext(workDirectory: work))104let extractSeconds = Date().timeIntervalSince(start)105let peak = peakRSS()106guard extracted.facts["bytes"] == "\(bytes)" else { print("extracted \(extracted.facts["bytes"] ?? "?") bytes, expected \(bytes)"); exit(1) }107108print(String(format: "BENCH format=%@ level=\(level) data=%.0fMB archive=%.1fMB ratio=%.2f create=%.2fs (%.0f MB/s) extract=%.2fs (%.0f MB/s) peakRSS(create)=%.0fMB peakRSS(process)=%.0fMB",109 format.rawValue, megabytesOfData, Double(archiveBytes) / 1_048_576, Double(bytes) / Double(max(archiveBytes, 1)), createSeconds, megabytesOfData / createSeconds,110 extractSeconds, megabytesOfData / extractSeconds, Double(createPeak) / 1_048_576, Double(peak) / 1_048_576))111