SPB Git

Repository Public License

Support.swift 11.2 KB 216 lines swift

1// ORIX Convert — Packages/EngineData/Tests/EngineDataTests/Support.swift2// Description: Test support: scratch directories, the canonical dataset, and fixture builders that write every3//              source format directly (not through the conversion pipeline under test).4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import Testing7import ConvertCore8@testable import EngineData910final class Scratch: @unchecked Sendable {11    let url: URL12    init(_ label: String = "t") {13        url = FileManager.default.temporaryDirectory.appendingPathComponent("orix-data-tests/\(label)-\(UUID().uuidString)", isDirectory: true)14        try? FileManager.default.createDirectory(at: url, withIntermediateDirectories: true)15    }16    deinit { try? FileManager.default.removeItem(at: url) }17    func file(_ name: String) -> URL { url.appendingPathComponent(name) }18    func dir(_ name: String) -> URL {19        let d = url.appendingPathComponent(name, isDirectory: true)20        try? FileManager.default.createDirectory(at: d, withIntermediateDirectories: true)21        return d22    }23}2425final class WarningBag: @unchecked Sendable {26    private let lock = NSLock()27    private var items: [ConversionWarning] = []28    private var fractions: [Double?] = []29    func add(_ w: ConversionWarning) { lock.lock(); items.append(w); lock.unlock() }30    func progress(_ p: ProgressUpdate) { lock.lock(); fractions.append(p.fraction); lock.unlock() }31    var all: [ConversionWarning] { lock.lock(); defer { lock.unlock() }; return items }32    var codes: Set<String> { Set(all.map(\.code)) }33    var progressValues: [Double?] { lock.lock(); defer { lock.unlock() }; return fractions }34}3536struct Converted {37    let output: ConversionOutput38    let warnings: WarningBag39    let request: ConversionRequest40    var file: URL { output.files[0] }41}4243/// Runs a conversion through the public engine API.44func runConversion(_ source: URL, _ sourceFormat: FormatID, to destination: FormatID, in scratch: Scratch, options: [String: OptionValue] = [:], name: String? = nil) async throws -> Converted {45    let engine = DataEngine()46    let work = scratch.dir("work-\(UUID().uuidString.prefix(8))")47    let request = ConversionRequest(source: source, sourceFormat: sourceFormat, destinationFormat: destination, options: ConversionOptions(options), outputBaseName: name)48    let bag = WarningBag()49    let context = ConversionContext(workDirectory: work, progress: { bag.progress($0) }, warn: { bag.add($0) })50    let output = try await engine.convert(request, context: context)51    // The engine must leave nothing but its outputs in the work directory.52    let left = try FileManager.default.contentsOfDirectory(atPath: work.path).filter { name in !output.files.contains { $0.lastPathComponent == name } }53    #expect(left.isEmpty, "leftovers in the work directory: \(left)")54    return Converted(output: output, warnings: bag, request: request)55}5657/// The canonical flat dataset: column names, kinds and rows.58enum Canonical {59    static let columns: [TableColumn] = [60        .init(name: "id", kind: .integer), .init(name: "name", kind: .string), .init(name: "big", kind: .integer), .init(name: "price", kind: .double),61        .init(name: "flag", kind: .boolean), .init(name: "day", kind: .date), .init(name: "at", kind: .timestamp), .init(name: "zip", kind: .string), .init(name: "note", kind: .string),62    ]63    static func day(_ y: Int64, _ m: Int64, _ d: Int64) -> Int32 { Int32(CivilDate.daysFromCivil(year: y, month: m, day: d)) }64    static func ts(_ y: Int64, _ mo: Int64, _ d: Int64, _ h: Int64, _ mi: Int64, _ s: Int64, _ us: Int64 = 0) -> Int64 {65        CivilDate.daysFromCivil(year: y, month: mo, day: d) * CivilDate.microsPerDay + (h * 3600 + mi * 60 + s) * 1_000_000 + us66    }67    static let rows: [[Cell]] = [68        [.int(1), .string("héllo wörld"), .int(9_007_199_254_740_993), .double(1.5), .bool(true), .date(day(2024, 2, 29)), .timestamp(ts(2024, 2, 29, 13, 45, 10, 123_000)), .string("00501"), .string("")],69        [.int(2), .string("quote \" and, comma; pipe|"), .int(-9_007_199_254_740_993), .double(-0.25), .bool(false), .date(day(1999, 12, 31)), .timestamp(ts(2001, 1, 1, 0, 0, 0)), .string("02134"), .null],70        [.int(3), .string("emoji 😀 and שלום עולם"), .int(0), .double(1234.5678), .null, .null, .null, .string("10001"), .string("x")],71        [.int(4), .string("line1\nline2"), .null, .null, .bool(true), .date(day(1970, 1, 1)), .timestamp(ts(1985, 7, 4, 23, 59, 59)), .null, .string(" padded ")],72    ]73    static var names: [String] { columns.map(\.name) }74    static var texts: [[String?]] { rows.map { $0.map(\.text) } }7576    static func csvField(_ s: String, delimiter: String) -> String {77        if s.isEmpty || s.contains("\"") || s.contains("\n") || s.contains(delimiter) || s.contains("\r") { return "\"" + s.replacingOccurrences(of: "\"", with: "\"\"") + "\"" }78        return s79    }8081    static func delimited(_ delimiter: String) -> String {82        var out = names.joined(separator: delimiter) + "\n"83        for r in rows { out += r.map { $0.text.map { csvField($0, delimiter: delimiter) } ?? "" }.joined(separator: delimiter) + "\n" }84        return out85    }8687    static func jsonValue(_ c: Cell) -> String {88        switch c {89        case .null: "null"90        case .int(let v): String(v)91        case .double(let d): "\(d)"92        case .bool(let b): b ? "true" : "false"93        default: JSONText.quote(c.text ?? "")94        }95    }9697    static func jsonRecords() -> [String] {98        rows.map { r in "{" + zip(names, r).map { "\(JSONText.quote($0)):\(jsonValue($1))" }.joined(separator: ",") + "}" }99    }100101    /// Writes the canonical dataset in `format` with a writer that is independent of the conversion pipeline.102    static func fixture(_ format: FormatID, in scratch: Scratch) async throws -> URL {103        let url = scratch.file("canonical-\(UUID().uuidString.prefix(6)).\(FormatCatalog.format(format)?.preferredExtension ?? format.rawValue)")104        switch format {105        case .csv: try delimited(",").write(to: url, atomically: true, encoding: .utf8)106        case .tsv: try delimited("\t").write(to: url, atomically: true, encoding: .utf8)107        case .json: try ("[\n" + jsonRecords().joined(separator: ",\n") + "\n]\n").write(to: url, atomically: true, encoding: .utf8)108        case .jsonl: try (jsonRecords().joined(separator: "\n") + "\n").write(to: url, atomically: true, encoding: .utf8)109        case .xlsx:110            let w = try XLSXWriter(url: url)111            try w.beginSheet(name: "Data", columns: columns, header: true)112            for r in rows { try w.writeRow(r) }113            try w.finish()114        case .ods:115            let w = try ODSWriter(url: url)116            try w.beginSheet(name: "Data", columns: columns, header: true)117            for r in rows { try w.writeRow(r) }118            try w.finish()119        case .xls:120            let x = try await fixture(.xlsx, in: scratch)121            let ws = try Workspace(parent: scratch.url)122            let produced = try LibreOffice.convert(x, to: "xls", workspace: ws, control: nil)123            try FileManager.default.moveItem(at: produced, to: url)124        case .sqlite:125            let s = try SQLiteSink(url: url)126            try s.beginTable(name: "data", columns: columns)127            for r in rows { try s.writeRow(r) }128            try s.finish()129        case .arrow:130            let w = try ArrowWriter(url: url, columns: columns, format: .file)131            try w.write(rows: rows)132            try w.finish()133        case .parquet, .duckdb:134            let ws = try Workspace(parent: scratch.url)135            let session = try Pipeline.makeSession(workspace: ws, sources: [], extraDirectories: [scratch.url], options: DataOptions(), control: nil)136            func lit(_ c: Cell, _ k: ColumnKind) throws -> String {137                guard let t = c.text else { return "NULL" }138                switch k {139                case .integer: return "CAST(\(t) AS BIGINT)"140                case .double: return "CAST(\(t) AS DOUBLE)"141                case .boolean: return t142                case .date: return "DATE \(try SQL.literal(t))"143                case .timestamp: return "TIMESTAMP \(try SQL.literal(t))"144                default: return try SQL.literal(t)145                }146            }147            let types = ["BIGINT", "VARCHAR", "BIGINT", "DOUBLE", "BOOLEAN", "DATE", "TIMESTAMP", "VARCHAR", "VARCHAR"]148            let values = try rows.map { r in "(" + (try zip(r, columns).map { try lit($0, $1.kind) }.joined(separator: ", ")) + ")" }.joined(separator: ", ")149            let select = "SELECT " + zip(names, types).enumerated().map { "CAST(col\($0.offset) AS \($0.element.1)) AS \"\($0.element.0)\"" }.joined(separator: ", ") + " FROM (VALUES \(values))"150            if format == .parquet { try session.execute("COPY (\(select)) TO \(try SQL.path(url)) (FORMAT parquet)") }151            else {152                try session.execute("ATTACH \(try SQL.path(url)) AS out")153                try session.execute("CREATE TABLE out.main.data AS \(select)")154                try session.execute("DETACH out")155            }156        case .xml:157            var s = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<rows>\n"158            for r in rows {159                s += "  <row>"160                for (n, c) in zip(names, r) { if let t = c.text { s += "<\(n)>\(xmlEscaped(t))</\(n)>" } }161                s += "</row>\n"162            }163            try (s + "</rows>\n").write(to: url, atomically: true, encoding: .utf8)164        case .yaml:165            var s = ""166            for r in rows {167                var first = true168                for (n, c) in zip(names, r) {169                    let v: String170                    switch c {171                    case .null: v = "null"172                    case .string(let t): v = JSONText.quote(t)173                    default: v = c.text ?? "null"174                    }175                    s += (first ? "- " : "  ") + "\(n): \(v)\n"; first = false176                }177            }178            try s.write(to: url, atomically: true, encoding: .utf8)179        case .toml:180            var s = ""181            for r in rows {182                s += "[[rows]]\n"183                for (n, c) in zip(names, r) {184                    switch c {185                    case .null: continue186                    case .string(let t): s += "\(n) = \(TOMLCodec.Writer.string(t))\n"187                    default: s += "\(n) = \(c.text ?? "")\n"188                    }189                }190                s += "\n"191            }192            try s.write(to: url, atomically: true, encoding: .utf8)193        case .plist:194            var list: [[String: Any]] = []195            for r in rows {196                var d: [String: Any] = [:]197                for (n, c) in zip(names, r) {198                    switch c {199                    case .null: continue200                    case .int(let v): d[n] = NSNumber(value: v)201                    case .double(let v): d[n] = NSNumber(value: v)202                    case .bool(let b): d[n] = NSNumber(value: b)203                    default: d[n] = c.text ?? ""204                    }205                }206                list.append(d)207            }208            try PropertyListSerialization.data(fromPropertyList: list, format: .xml, options: 0).write(to: url)209        default: Issue.record("no fixture for \(format)")210        }211        return url212    }213}214215let libreOfficeAvailable = LibreOffice.isAvailable216
Files main
Menu

Theme