Repository Public License
1// ORIX Convert — Packages/EngineData/Tests/EngineDataTests/Support.swift2// Description: Test support: scratch directories, the canonical dataset, and fixture builders that write every3// source format directly (not through the conversion pipeline under test).4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import Testing7import ConvertCore8@testable import EngineData910final class Scratch: @unchecked Sendable {11 let url: URL12 init(_ label: String = "t") {13 url = FileManager.default.temporaryDirectory.appendingPathComponent("orix-data-tests/\(label)-\(UUID().uuidString)", isDirectory: true)14 try? FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)15 }16 deinit { try? FileManager.default.removeItem(at: url) }17 func file(_ name: String) -> URL { url.appendingPathComponent(name) }18 func dir(_ name: String) -> URL {19 let d = url.appendingPathComponent(name, isDirectory: true)20 try? FileManager.default.createDirectory(at: d, withIntermediateDirectories: true)21 return d22 }23}2425final class WarningBag: @unchecked Sendable {26 private let lock = NSLock()27 private var items: [ConversionWarning] = []28 private var fractions: [Double?] = []29 func add(_ w: ConversionWarning) { lock.lock(); items.append(w); lock.unlock() }30 func progress(_ p: ProgressUpdate) { lock.lock(); fractions.append(p.fraction); lock.unlock() }31 var all: [ConversionWarning] { lock.lock(); defer { lock.unlock() }; return items }32 var codes: Set<String> { Set(all.map(\.code)) }33 var progressValues: [Double?] { lock.lock(); defer { lock.unlock() }; return fractions }34}3536struct Converted {37 let output: ConversionOutput38 let warnings: WarningBag39 let request: ConversionRequest40 var file: URL { output.files[0] }41}4243/// Runs a conversion through the public engine API.44func runConversion(_ source: URL, _ sourceFormat: FormatID, to destination: FormatID, in scratch: Scratch, options: [String: OptionValue] = [:], name: String? = nil) async throws -> Converted {45 let engine = DataEngine()46 let work = scratch.dir("work-\(UUID().uuidString.prefix(8))")47 let request = ConversionRequest(source: source, sourceFormat: sourceFormat, destinationFormat: destination, options: ConversionOptions(options), outputBaseName: name)48 let bag = WarningBag()49 let context = ConversionContext(workDirectory: work, progress: { bag.progress($0) }, warn: { bag.add($0) })50 let output = try await engine.convert(request, context: context)51 // The engine must leave nothing but its outputs in the work directory.52 let left = try FileManager.default.contentsOfDirectory(atPath: work.path).filter { name in !output.files.contains { $0.lastPathComponent == name } }53 #expect(left.isEmpty, "leftovers in the work directory: \(left)")54 return Converted(output: output, warnings: bag, request: request)55}5657/// The canonical flat dataset: column names, kinds and rows.58enum Canonical {59 static let columns: [TableColumn] = [60 .init(name: "id", kind: .integer), .init(name: "name", kind: .string), .init(name: "big", kind: .integer), .init(name: "price", kind: .double),61 .init(name: "flag", kind: .boolean), .init(name: "day", kind: .date), .init(name: "at", kind: .timestamp), .init(name: "zip", kind: .string), .init(name: "note", kind: .string),62 ]63 static func day(_ y: Int64, _ m: Int64, _ d: Int64) -> Int32 { Int32(CivilDate.daysFromCivil(year: y, month: m, day: d)) }64 static func ts(_ y: Int64, _ mo: Int64, _ d: Int64, _ h: Int64, _ mi: Int64, _ s: Int64, _ us: Int64 = 0) -> Int64 {65 CivilDate.daysFromCivil(year: y, month: mo, day: d) * CivilDate.microsPerDay + (h * 3600 + mi * 60 + s) * 1_000_000 + us66 }67 static let rows: [[Cell]] = [68 [.int(1), .string("héllo wörld"), .int(9_007_199_254_740_993), .double(1.5), .bool(true), .date(day(2024, 2, 29)), .timestamp(ts(2024, 2, 29, 13, 45, 10, 123_000)), .string("00501"), .string("")],69 [.int(2), .string("quote \" and, comma; pipe|"), .int(-9_007_199_254_740_993), .double(-0.25), .bool(false), .date(day(1999, 12, 31)), .timestamp(ts(2001, 1, 1, 0, 0, 0)), .string("02134"), .null],70 [.int(3), .string("emoji 😀 and שלום עולם"), .int(0), .double(1234.5678), .null, .null, .null, .string("10001"), .string("x")],71 [.int(4), .string("line1\nline2"), .null, .null, .bool(true), .date(day(1970, 1, 1)), .timestamp(ts(1985, 7, 4, 23, 59, 59)), .null, .string(" padded ")],72 ]73 static var names: [String] { columns.map(\.name) }74 static var texts: [[String?]] { rows.map { $0.map(\.text) } }7576 static func csvField(_ s: String, delimiter: String) -> String {77 if s.isEmpty || s.contains("\"") || s.contains("\n") || s.contains(delimiter) || s.contains("\r") { return "\"" + s.replacingOccurrences(of: "\"", with: "\"\"") + "\"" }78 return s79 }8081 static func delimited(_ delimiter: String) -> String {82 var out = names.joined(separator: delimiter) + "\n"83 for r in rows { out += r.map { $0.text.map { csvField($0, delimiter: delimiter) } ?? "" }.joined(separator: delimiter) + "\n" }84 return out85 }8687 static func jsonValue(_ c: Cell) -> String {88 switch c {89 case .null: "null"90 case .int(let v): String(v)91 case .double(let d): "\(d)"92 case .bool(let b): b ? "true" : "false"93 default: JSONText.quote(c.text ?? "")94 }95 }9697 static func jsonRecords() -> [String] {98 rows.map { r in "{" + zip(names, r).map { "\(JSONText.quote($0)):\(jsonValue($1))" }.joined(separator: ",") + "}" }99 }100101 /// Writes the canonical dataset in `format` with a writer that is independent of the conversion pipeline.102 static func fixture(_ format: FormatID, in scratch: Scratch) async throws -> URL {103 let url = scratch.file("canonical-\(UUID().uuidString.prefix(6)).\(FormatCatalog.format(format)?.preferredExtension ?? format.rawValue)")104 switch format {105 case .csv: try delimited(",").write(to: url, atomically: true, encoding: .utf8)106 case .tsv: try delimited("\t").write(to: url, atomically: true, encoding: .utf8)107 case .json: try ("[\n" + jsonRecords().joined(separator: ",\n") + "\n]\n").write(to: url, atomically: true, encoding: .utf8)108 case .jsonl: try (jsonRecords().joined(separator: "\n") + "\n").write(to: url, atomically: true, encoding: .utf8)109 case .xlsx:110 let w = try XLSXWriter(url: url)111 try w.beginSheet(name: "Data", columns: columns, header: true)112 for r in rows { try w.writeRow(r) }113 try w.finish()114 case .ods:115 let w = try ODSWriter(url: url)116 try w.beginSheet(name: "Data", columns: columns, header: true)117 for r in rows { try w.writeRow(r) }118 try w.finish()119 case .xls:120 let x = try await fixture(.xlsx, in: scratch)121 let ws = try Workspace(parent: scratch.url)122 let produced = try LibreOffice.convert(x, to: "xls", workspace: ws, control: nil)123 try FileManager.default.moveItem(at: produced, to: url)124 case .sqlite:125 let s = try SQLiteSink(url: url)126 try s.beginTable(name: "data", columns: columns)127 for r in rows { try s.writeRow(r) }128 try s.finish()129 case .arrow:130 let w = try ArrowWriter(url: url, columns: columns, format: .file)131 try w.write(rows: rows)132 try w.finish()133 case .parquet, .duckdb:134 let ws = try Workspace(parent: scratch.url)135 let session = try Pipeline.makeSession(workspace: ws, sources: [], extraDirectories: [scratch.url], options: DataOptions(), control: nil)136 func lit(_ c: Cell, _ k: ColumnKind) throws -> String {137 guard let t = c.text else { return "NULL" }138 switch k {139 case .integer: return "CAST(\(t) AS BIGINT)"140 case .double: return "CAST(\(t) AS DOUBLE)"141 case .boolean: return t142 case .date: return "DATE \(try SQL.literal(t))"143 case .timestamp: return "TIMESTAMP \(try SQL.literal(t))"144 default: return try SQL.literal(t)145 }146 }147 let types = ["BIGINT", "VARCHAR", "BIGINT", "DOUBLE", "BOOLEAN", "DATE", "TIMESTAMP", "VARCHAR", "VARCHAR"]148 let values = try rows.map { r in "(" + (try zip(r, columns).map { try lit($0, $1.kind) }.joined(separator: ", ")) + ")" }.joined(separator: ", ")149 let select = "SELECT " + zip(names, types).enumerated().map { "CAST(col\($0.offset) AS \($0.element.1)) AS \"\($0.element.0)\"" }.joined(separator: ", ") + " FROM (VALUES \(values))"150 if format == .parquet { try session.execute("COPY (\(select)) TO \(try SQL.path(url)) (FORMAT parquet)") }151 else {152 try session.execute("ATTACH \(try SQL.path(url)) AS out")153 try session.execute("CREATE TABLE out.main.data AS \(select)")154 try session.execute("DETACH out")155 }156 case .xml:157 var s = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rows>\n"158 for r in rows {159 s += " <row>"160 for (n, c) in zip(names, r) { if let t = c.text { s += "<\(n)>\(xmlEscaped(t))</\(n)>" } }161 s += "</row>\n"162 }163 try (s + "</rows>\n").write(to: url, atomically: true, encoding: .utf8)164 case .yaml:165 var s = ""166 for r in rows {167 var first = true168 for (n, c) in zip(names, r) {169 let v: String170 switch c {171 case .null: v = "null"172 case .string(let t): v = JSONText.quote(t)173 default: v = c.text ?? "null"174 }175 s += (first ? "- " : " ") + "\(n): \(v)\n"; first = false176 }177 }178 try s.write(to: url, atomically: true, encoding: .utf8)179 case .toml:180 var s = ""181 for r in rows {182 s += "[[rows]]\n"183 for (n, c) in zip(names, r) {184 switch c {185 case .null: continue186 case .string(let t): s += "\(n) = \(TOMLCodec.Writer.string(t))\n"187 default: s += "\(n) = \(c.text ?? "")\n"188 }189 }190 s += "\n"191 }192 try s.write(to: url, atomically: true, encoding: .utf8)193 case .plist:194 var list: [[String: Any]] = []195 for r in rows {196 var d: [String: Any] = [:]197 for (n, c) in zip(names, r) {198 switch c {199 case .null: continue200 case .int(let v): d[n] = NSNumber(value: v)201 case .double(let v): d[n] = NSNumber(value: v)202 case .bool(let b): d[n] = NSNumber(value: b)203 default: d[n] = c.text ?? ""204 }205 }206 list.append(d)207 }208 try PropertyListSerialization.data(fromPropertyList: list, format: .xml, options: 0).write(to: url)209 default: Issue.record("no fixture for \(format)")210 }211 return url212 }213}214215let libreOfficeAvailable = LibreOffice.isAvailable216