SPB Git

Repository Public License

FlattenTests.swift 27.6 KB 308 lines swift

1// ORIX Convert — Packages/EngineScientific/Tests/EngineScientificTests/FlattenTests.swift2// Description: Flattening to CSV, JSON and Parquet with exact expected tables for every layout, subsetting with3//              strides, fill / packed / time handling, the refusals that make flattening explicit, and the4//              metadata that records the transformation.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import Testing8import ConvertCore9import CHDF510import CHDF5Shim11@testable import EngineScientific1213func lines(_ text: String) -> [String] { text.components(separatedBy: "\r\n").filter { !$0.isEmpty } }1415@Suite struct FlattenCSVTests {16    @Test func singleDatasetNeedsNoSelection() async throws {17        let temp = TempDir()18        let r = try await convert(Fixture.single(temp.file("single.h5")), .hdf5, to: .csv, in: temp)19        // 3 × 4 values without dimension names: long layout, index columns dim0 / dim1, C order.20        #expect(lines(try r.text()) == ["dim0,dim1,m", "0,0,0", "0,1,1", "0,2,2", "0,3,3", "1,0,4", "1,1,5", "1,2,6", "1,3,7", "2,0,8", "2,1,9", "2,2,10", "2,3,11"])21        #expect(r.output.facts["sci.layout"] == "long" && r.output.facts["sci.rows"] == "12" && r.output.facts["sci.columns"] == "3")22        #expect(r.output.facts["sci.variables"] == "/m")23        let w = try #require(r.warnings.first("sci-flattened"))24        #expect(w.message == "The table is a flattened view (long layout, 12 row(s)) of /m (? = 3 × ? = 4): the original multidimensional structure is described in the output's metadata, not kept.")25        #expect(r.progress.fractions.last == 1)26    }2728    @Test func severalDatasetsMustBeChosen() async throws {29        let temp = TempDir()30        let source = Fixture.everything(in: temp.url)31        let message = await failure(source, .hdf5, to: .csv, in: temp)32        #expect(message.hasPrefix("A setting is not valid. The file holds 23 datasets; choose which to convert in “Variables”. Available: /array/vectors [int32[3], 2]; /chunked/grid [int16, 20 × 30]; /compound/particles [compound {id: int32, mass: float64, name: string(6), flags: uint8}, 3];"), "\(message)")33        #expect(message.contains("/scalar [float64, ]") && message.contains("/unlimited/series [float32, 7]"))34        // References are not candidates; coordinate variables are not offered by default.35        #expect(!message.contains("/refs/objref") && !message.contains("/dims/x"))36        // The same message reaches the app before converting.37        let pre = ScientificEngine().preflight(ConversionRequest(source: source, sourceFormat: .hdf5, destinationFormat: .csv), inspection: nil)38        #expect(pre.count == 1 && pre[0].code == "sci-selection-needed" && pre[0].message.contains("choose which to convert"))39        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/nope")]) == "A setting is not valid. The file has no object at /nope.")40        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/ints")]) == "A setting is not valid. /ints is not a dataset.")41        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/refs/objref")]) == "A setting is not valid. /refs/objref holds reference, which cannot be written as table cells.")42        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/ints/i8, /chunked/grid")]).contains("must share the same dimensions: /ints/i8 (? = 4) and /chunked/grid (y = 20, x = 30) do not"))43    }4445    @Test func longLayoutWithCoordinatesSubsetAndStride() async throws {46        let temp = TempDir()47        let source = Fixture.everything(in: temp.url)48        let r = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/chunked/grid"), "subset": .string("y=0:2, x=0:3")])49        #expect(lines(try r.text()) == ["y,x,grid", "0,10.0,-999", "0,10.5,1", "0,11.0,2", "2,10.0,100", "2,10.5,101", "2,11.0,102"])50        // Strides, an open end and a dimension given by position.51        let s = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("chunked/grid"), "subset": .string("y=17::2; 1=28:")])52        #expect(lines(try s.text()) == ["y,x,grid", "34,24.0,1728", "34,24.5,1729", "38,24.0,1928", "38,24.5,1929"])53        // The values are stored packed and with a fill value: both facts are said, nothing is applied silently.54        #expect(r.warnings.first("sci-packed-raw")?.message == "/chunked/grid stores packed values (scale_factor 0.01, add_offset 273.15). The table holds the stored numbers, not the physical quantity; choose “Unpack” to apply them.")55        #expect(r.warnings.first("sci-fill-kept")?.message.contains("marks missing data with -999") == true)56        #expect(r.warnings.first("sci-attributes-not-carried")?.message.hasPrefix("10 attribute(s)") == true)57        // Explicit handling.58        let u = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/chunked/grid"), "subset": .string("y=0:1,x=0:3"), "fill": .string("null"), "packed": .string("unpack")])59        #expect(lines(try u.text()) == ["y,x,grid", "0,10.0,", "0,10.5,\(1 * 0.01 + 273.15)", "0,11.0,\(2 * 0.01 + 273.15)"])60        #expect(u.output.facts["sci.nulls"] == "1" && u.warnings.first("sci-packed-raw") == nil && u.warnings.first("sci-fill-kept") == nil)61        let transformation = try #require(u.output.facts["sci.transformation"])62        #expect(transformation == "{\"source_format\":\"HDF5\",\"layout\":\"long: one row per cell, one column per dimension, then one column per variable\",\"row_order\":\"C order: the last dimension varies fastest\",\"rows\":3,\"variables\":[{\"path\":\"/chunked/grid\",\"type\":\"int16\",\"dimensions\":[\"y\",\"x\"],\"shape\":[20,30],\"units\":\"K\",\"packed\":\"unpacked: value × 0.01 + 273.15, written as float64\",\"fill_values\":\"converted to null (-999.0)\"}],\"dimensions\":[{\"name\":\"y\",\"length\":20,\"start\":0,\"count\":1,\"stride\":1,\"labels\":\"values of /dims/y\"},{\"name\":\"x\",\"length\":30,\"start\":0,\"count\":3,\"stride\":1,\"labels\":\"values of /dims/x\"}]}")63    }6465    @Test func wideLayout() async throws {66        let temp = TempDir()67        let source = Fixture.everything(in: temp.url)68        let r = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/chunked/grid"), "layout": .string("wide"), "subset": .string("y=0:3,x=0:4")])69        #expect(lines(try r.text()) == ["y,x=10.0,x=10.5,x=11.0,x=11.5", "0,-999,1,2,3", "2,100,101,102,103", "4,200,201,202,203"])70        #expect(r.output.facts["sci.rows"] == "3" && r.output.facts["sci.columns"] == "5")71        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/ints/i8"), "layout": .string("wide")]).contains("The wide layout needs exactly one two-dimensional variable"))72        // The whole grid: 20 rows of 30 values.73        let full = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/chunked/grid"), "layout": .string("wide")])74        let all = lines(try full.text())75        #expect(all.count == 21 && all[20] == "38," + (0..<30).map { String(1900 + $0) }.joined(separator: ","))76    }7778    @Test func tableLayouts() async throws {79        let temp = TempDir()80        let source = Fixture.everything(in: temp.url)81        // A compound dataset: one column per member, one row per element.82        let c = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/compound/particles")])83        #expect(lines(try c.text()) == ["particles.id,particles.mass,particles.name,particles.flags", "1,1.5,p1,1", "2,2.25,proton,200", "-3,0.0,,0"])84        #expect(c.output.facts["sci.layout"] == "table")85        // One-dimensional variables sharing a dimension: a table; with the long layout, an index column too.86        let t = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/ints/i8,/ints/u64,/ints/i64")])87        #expect(lines(try t.text()) == ["i8,u64,i64", "-128,0,-9223372036854775808", "-1,1,-1", "0,9223372036854775808,0", "127,18446744073709551615,9223372036854775807"])88        let l = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/ints/i8,/ints/u8"), "layout": .string("long"), "subset": .string("dim0=1:4:2")])89        #expect(lines(try l.text()) == ["dim0,i8,u8", "1,-1,1", "3,127,255"])90        // Strings, enumerations, array elements, a scalar, special floats.91        #expect(lines(try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/strings/variable")]).text()) == ["variable", "one", "deux é"])92        #expect(lines(try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/enum/colors")]).text()) == ["colors", "RED", "BLUE", "GREEN", "BLUE"])93        #expect(lines(try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/array/vectors")]).text()) == ["vectors", "\"[1,2,3]\"", "\"[4,5,6]\""])94        #expect(lines(try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/scalar")]).text()) == ["scalar", "42.5"])95        #expect(lines(try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/floats/f32"), "layout": .string("long")]).text()) == ["dim0,dim1,f32", "0,0,0.1", "0,1,-1.5", "0,2,3.25", "1,0,1e-10", "1,1,1e+20", "1,2,NaN"])96        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/chunked/grid"), "layout": .string("table")]).contains("The table layout (no dimension columns) is for one-dimensional variables"))97    }9899    @Test func timeDecodingIsOptInAndCalendarAware() async throws {100        let temp = TempDir()101        let source = Fixture.everything(in: temp.url)102        // Default: raw numbers, the coordinate column kept because the dimension has coordinate values.103        let raw = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/time/values")])104        #expect(lines(try raw.text()) == ["t,values", "0.0,1", "1.5,2", "24.0,-9999", "48.0,4", "100000.0,5"])105        let decoded = try await convert(source, .hdf5, to: .csv, in: temp, options: ["variables": .string("/time/values"), "time.decode": .bool(true), "fill": .string("null")])106        #expect(lines(try decoded.text()) == ["t,values", "2000-01-01T00:00:00Z,1", "2000-01-01T01:30:00Z,2", "2000-01-02T00:00:00Z,", "2000-01-03T00:00:00Z,4", "2011-05-29T16:00:00Z,5"])107        #expect(decoded.output.facts["sci.transformation"]?.contains("\"labels\":\"values of /time/t, decoded to ISO 8601 UTC from hours since 2000-01-01 00:00:00\"") == true)108        // CF time parsing and the calendars that are refused.109        #expect(CFTime(units: "seconds since 1970-01-01T00:00:00Z")?.text(86_400.5) == "1970-01-02T00:00:00.5Z")110        #expect(CFTime(units: "days since 1900-01-01")?.text(36_524) == "2000-01-01T00:00:00Z")111        #expect(CFTime(units: "minutes since 2020-02-28 23:00:00 -05:00")?.text(60 * 25) == "2020-03-01T05:00:00Z")112        #expect(CFTime(units: "hours since 2000-01-01 00:00:00")?.text(.nan) == nil)113        #expect(CFTime(units: "months since 2000-01-01") == nil && CFTime(units: "K") == nil && CFTime(units: "days since yesterday") == nil)114        var w: [ConversionWarning] = []115        var c = SciConventions()116        c.units = "days since 2000-01-01"; c.calendar = "noleap"117        #expect(FlattenPlan.testTimeAxis(c, warnings: &w) == nil && w.first?.message.contains("uses the calendar “noleap”, which this app does not decode") == true)118        c.calendar = "360_day"119        #expect(FlattenPlan.testTimeAxis(c, warnings: &w) == nil)120        c.calendar = nil; c.units = "days since 1500-01-01"121        #expect(FlattenPlan.testTimeAxis(c, warnings: &w) == nil && w.last?.message.contains("before 15 October 1582") == true)122        c.calendar = "proleptic_gregorian"123        #expect(FlattenPlan.testTimeAxis(c, warnings: &w)?.text(0) == "1500-01-01T00:00:00Z")124    }125126    @Test func rowLimitAndSubsetErrors() async throws {127        let temp = TempDir()128        let source = Fixture.everything(in: temp.url)129        let grid: [String: OptionValue] = ["variables": .string("/chunked/grid")]130        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: grid.merging(["rows.limit": .int(599)]) { $1 }) == "A setting is not valid. The table would have 600 rows; the row limit is 599. Subset the dimensions, or raise “Row limit” to confirm such a large table.")131        _ = try await convert(source, .hdf5, to: .csv, in: temp, options: grid.merging(["rows.limit": .int(600)]) { $1 })132        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: grid.merging(["subset": .string("z=0:2")]) { $1 }).contains("The subset names z, which is not a dimension of the selected variables. Dimensions: y, x."))133        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: grid.merging(["subset": .string("y=a:b")]) { $1 }).contains("is not a non-negative whole number"))134        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: grid.merging(["subset": .string("y=0:5:0")]) { $1 }).contains("The stride of y must be at least 1."))135        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: grid.merging(["subset": .string("y=99:")]) { $1 }).contains("starts at 99, past its length 20"))136        #expect(await failure(source, .hdf5, to: .csv, in: temp, options: grid.merging(["subset": .string("y")]) { $1 }).contains("is not a subset"))137        // An empty selection is a header and nothing else; a stop past the end is clipped.138        let empty = try await convert(source, .hdf5, to: .csv, in: temp, options: grid.merging(["subset": .string("y=5:5")]) { $1 })139        #expect(lines(try empty.text()) == ["y,x,grid"] && empty.output.facts["sci.rows"] == "0")140        let clipped = try await convert(source, .hdf5, to: .csv, in: temp, options: grid.merging(["subset": .string("y=19:500,x=29")]) { $1 })141        #expect(lines(try clipped.text()) == ["y,x,grid", "38,24.5,1929"])142    }143144    @Test func sidecarKeepsAttributes() async throws {145        let temp = TempDir()146        let r = try await convert(Fixture.everything(in: temp.url), .hdf5, to: .csv, in: temp, options: ["variables": .string("/chunked/grid"), "subset": .string("y=0:1,x=0:1"), "metadata.sidecar": .bool(true)], name: "grid")147        #expect(try FileManager.default.contentsOfDirectory(atPath: r.file.path).sorted() == ["grid.csv", "grid.metadata.json"])148        #expect(lines(try r.text("grid.csv")) == ["y,x,grid", "0,10.0,-999"])149        let meta = try #require(try JSONSerialization.jsonObject(with: Data(contentsOf: r.file.appendingPathComponent("grid.metadata.json"))) as? [String: Any])150        let attributes = try #require(meta["orix_convert.attributes"] as? [String: [String: Any]])151        #expect(attributes["/"]?["title"] as? String == "Everything" && attributes["/"]?["version"] as? Int == 3)152        #expect(attributes["/"]?["ratios"] as? [Double] == [0.5, 1.5, 2.5] && attributes["/"]?["tags"] as? [String] == ["alpha", "βeta"])153        #expect(attributes["/chunked/grid"]?["scale_factor"] as? Double == 0.01 && attributes["/dims/x"]?["units"] as? String == "km")154        #expect((meta["orix_convert.transformation"] as? [String: Any])?["rows"] as? Int == 1)155        #expect(meta["orix_convert.source"] as? String == "everything.h5")156        #expect(r.warnings.first("sci-attributes-not-carried") == nil)157    }158159    @Test func netCDFSources() async throws {160        let temp = TempDir()161        let classic = Fixture.classic(temp.file("c.nc"))162        #expect(await failure(classic, .netcdf, to: .csv, in: temp).contains("The file holds 4 datasets; choose which to convert in “Variables”. Available: /station [char, 2 × 4]; /temp [int16, 3 × 2 × 3]; /scalarv [float64, ]; /pres [float32, 3]."))163        // Record variable, coordinates of three dimensions, strides, decoded time, fill → null.164        let r = try await convert(classic, .netcdf, to: .csv, in: temp, options: ["variables": .string("/temp"), "subset": .string("time=1:3,lon=0:3:2"), "time.decode": .bool(true), "fill": .string("null")])165        #expect(lines(try r.text()) == ["time,lat,lon,temp", "2020-01-01T06:00:00Z,45.0,-75.0,100", "2020-01-01T06:00:00Z,45.0,-74.0,", "2020-01-01T06:00:00Z,46.5,-75.0,110", "2020-01-01T06:00:00Z,46.5,-74.0,112",166                                         "2020-01-01T12:00:00Z,45.0,-75.0,200", "2020-01-01T12:00:00Z,45.0,-74.0,202", "2020-01-01T12:00:00Z,46.5,-75.0,210", "2020-01-01T12:00:00Z,46.5,-74.0,212"])167        // Characters along the last dimension are one text per row.168        let s = try await convert(classic, .netcdf, to: .csv, in: temp, options: ["variables": .string("/station")])169        #expect(lines(try s.text()) == ["lat,station", "45.0,ABCD", "46.5,EF"])170        // Two record variables sharing the record dimension.171        let p = try await convert(classic, .netcdf, to: .csv, in: temp, options: ["variables": .string("/pres,/time"), "layout": .string("table")])172        #expect(lines(try p.text()) == ["pres,time", "1000.5,0", "999.25,6", "998.0,12"])173        // NetCDF-4: an unlimited dimension without coordinate values is labelled by its index.174        let nc4 = Fixture.netCDF4(temp.file("raw.nc"))175        let t = try await convert(nc4, .netcdf, to: .csv, in: temp, options: ["variables": .string("/temp")])176        #expect(lines(try t.text()) == ["time,x,temp", "0,1.5,10.0", "0,2.5,11.0", "0,3.5,12.0", "1,1.5,20.0", "1,2.5,21.0", "1,3.5,22.0"])177        #expect(lines(try await convert(nc4, .netcdf, to: .csv, in: temp, options: ["variables": .string("/label")]).text()) == ["label", "abc"])178        #expect(lines(try await convert(nc4, .netcdf, to: .csv, in: temp, options: ["variables": .string("/model/bias")]).text()) == ["x,bias", "1.5,0.1", "2.5,0.2", "3.5,0.3"])179    }180}181182extension FlattenPlan {183    /// Exposes the calendar rules to the tests.184    static func testTimeAxis(_ c: SciConventions, warnings: inout [ConversionWarning]) -> CFTime? { timeAxisForTests(c, warnings: &warnings) }185}186187@Suite struct FlattenJSONTests {188    @Test func records() async throws {189        let temp = TempDir()190        let source = Fixture.everything(in: temp.url)191        let r = try await convert(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/time/values"), "fill": .string("null")])192        let text = try r.text()193        #expect(text.hasSuffix("\"columns\":[{\"name\":\"t\",\"type\":\"float64\"},{\"name\":\"values\",\"type\":\"int32\"}],\n\"rows\":[\n{\"t\":0.0,\"values\":1},\n{\"t\":1.5,\"values\":2},\n{\"t\":24.0,\"values\":null},\n{\"t\":48.0,\"values\":4},\n{\"t\":100000.0,\"values\":5}\n]}\n"))194        let object = try #require(try JSONSerialization.jsonObject(with: Data(text.utf8)) as? [String: Any])195        let meta = try #require(object["metadata"] as? [String: Any])196        #expect((meta["orix_convert.transformation"] as? [String: Any])?["layout"] as? String == "long: one row per cell, one column per dimension, then one column per variable")197        #expect(((meta["orix_convert.attributes"] as? [String: Any])?["/time/t"] as? [String: Any])?["calendar"] as? String == "standard")198        // Non-finite numbers cannot be written in JSON: null, counted, and said.199        let f = try await convert(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/floats/f32,/floats/f64")])200        #expect(try f.text().hasSuffix("{\"dim0\":1,\"dim1\":1,\"f32\":1e+20,\"f64\":1e+300},\n{\"dim0\":1,\"dim1\":2,\"f32\":null,\"f64\":null}\n]}\n"))201        #expect(f.warnings.first("sci-json-non-finite")?.message.hasPrefix("2 NaN or infinite value(s)") == true)202        // Text with quotes and unicode, 64-bit extremes.203        let s = try await convert(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/strings/variable")])204        #expect(try s.text().hasSuffix("\"rows\":[\n{\"variable\":\"one\"},\n{\"variable\":\"deux é\"},\n{\"variable\":\"\"}\n]}\n"))205        let u = try await convert(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/ints/u64")])206        #expect(try u.text().contains("{\"u64\":18446744073709551615}"))207    }208209    @Test func nestedArrays() async throws {210        let temp = TempDir()211        let source = Fixture.everything(in: temp.url)212        let r = try await convert(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/chunked/grid"), "json.shape": .string("nested"), "subset": .string("y=0:2,x=0:3")])213        #expect(try r.text().hasSuffix("\"variable\":\"/chunked/grid\",\n\"dimensions\":[\"y\",\"x\"],\n\"shape\":[2,3],\n\"data\":[[-999,1,2],[100,101,102]]}\n"))214        // Three dimensions from a classic file, with a null for the fill value.215        let classic = Fixture.classic(temp.file("c.nc"))216        let c = try await convert(classic, .netcdf, to: .json, in: temp, options: ["variables": .string("/temp"), "json.shape": .string("nested"), "fill": .string("null"), "subset": .string("time=0:2")])217        #expect(try c.text().hasSuffix("\"shape\":[2,2,3],\n\"data\":[[[0,1,2],[10,11,12]],[[100,101,null],[110,111,112]]]}\n"))218        // A scalar, an empty selection, a one-dimensional variable.219        #expect(try await convert(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/scalar"), "json.shape": .string("nested")]).text().hasSuffix("\"dimensions\":[],\n\"shape\":[],\n\"data\":42.5}\n"))220        #expect(try await convert(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/chunked/grid"), "json.shape": .string("nested"), "subset": .string("x=3:3")]).text().hasSuffix("\"shape\":[20,0],\n\"data\":[]}\n"))221        #expect(try await convert(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/ints/i16"), "json.shape": .string("nested")]).text().hasSuffix("\"data\":[-32768,-1,0,32767]}\n"))222        #expect(await failure(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/compound/particles"), "json.shape": .string("nested")]).contains("Nested JSON arrays hold one variable that is not a compound"))223        #expect(await failure(source, .hdf5, to: .json, in: temp, options: ["variables": .string("/chunked/grid"), "json.shape": .string("nested"), "rows.limit": .int(10)]).contains("the row limit is 10"))224    }225}226227@Suite struct FlattenParquetTests {228    @Test func typesNullsAndMetadata() async throws {229        let temp = TempDir()230        let source = Fixture.everything(in: temp.url)231        let r = try await convert(source, .hdf5, to: .parquet, in: temp, options: ["variables": .string("/ints/i8,/ints/u8,/ints/i16,/ints/u16,/ints/i32,/ints/u32,/ints/i64,/ints/u64")])232        let p = try ParquetFile(url: r.file)233        #expect(p.rows == 4 && p.createdBy == "ORIX Convert (EngineScientific)")234        #expect(p.columns.map(\.name) == ["i8", "u8", "i16", "u16", "i32", "u32", "i64", "u64"])235        // Physical INT32 (1) up to 32 bits, INT64 (2) beyond; converted types INT_8…INT_64 = 15…18, UINT_8…UINT_64 = 11…14.236        #expect(p.columns.map(\.physical) == [1, 1, 1, 1, 1, 1, 2, 2])237        #expect(p.columns.map(\.converted) == [15, 11, 16, 12, 17, 13, 18, 14])238        #expect(p.columns.allSatisfy { !$0.optional })239        func column(_ c: Int) throws -> [String] { let v = try p.read(column: c, group: 0); return (0..<v.values.count).map { v.values.text($0) } }240        #expect(try column(0) == ["-128", "-1", "0", "127"] && column(1) == ["0", "1", "128", "255"])241        #expect(try column(5) == ["0", "1", "2147483648", "4294967295"])242        #expect(try column(6) == ["-9223372036854775808", "-1", "0", "9223372036854775807"])243        #expect(try column(7) == ["0", "1", "9223372036854775808", "18446744073709551615"])244        #expect(p.metadata["orix_convert.transformation"] == r.output.facts["sci.transformation"])245        #expect(p.metadata["orix_convert.source"] == "\"everything.h5\"")246        #expect(p.metadata["orix_convert.attributes"]?.contains("\"/ints/i8\"") == false && p.metadata["orix_convert.attributes"]?.contains("\"title\":\"Everything\"") == true)247        // Nullable columns, floats, strings, gzip pages.248        for compression in ["none", "gzip"] {249            let n = try await convert(source, .hdf5, to: .parquet, in: temp, options: ["variables": .string("/time/values"), "fill": .string("null"), "time.decode": .bool(true), "parquet.compression": .string(compression)])250            let q = try ParquetFile(url: n.file)251            #expect(q.columns.map(\.name) == ["t", "values"] && q.columns.map(\.physical) == [6, 1] && q.columns.map(\.optional) == [true, true])252            let t = try q.read(column: 0, group: 0), v = try q.read(column: 1, group: 0)253            #expect((0..<5).map { t.values.text($0) } == ["2000-01-01T00:00:00Z", "2000-01-01T01:30:00Z", "2000-01-02T00:00:00Z", "2000-01-03T00:00:00Z", "2011-05-29T16:00:00Z"])254            #expect(v.valid == [true, true, false, true, true] && v.values.text(3) == "4")255            #expect(n.output.facts["parquet.compression"] == compression && n.output.facts["sci.nulls"] == "1")256        }257        let f = try await convert(source, .hdf5, to: .parquet, in: temp, options: ["variables": .string("/floats/f32,/floats/f64")])258        let fp = try ParquetFile(url: f.file)259        #expect(fp.columns.map(\.physical) == [2, 2, 4, 5] && fp.rows == 6)260        let f32 = try fp.read(column: 2, group: 0), f64 = try fp.read(column: 3, group: 0)261        guard case .float(let a) = f32.values, case .double(let b) = f64.values else { Issue.record("wrong column types"); return }262        #expect(a.prefix(5) == Fixture.f32.prefix(5) && a[5].isNaN && b == Fixture.f64)263        let c = try await convert(source, .hdf5, to: .parquet, in: temp, options: ["variables": .string("/compound/particles")])264        let cp = try ParquetFile(url: c.file)265        #expect(cp.columns.map(\.name) == ["particles.id", "particles.mass", "particles.name", "particles.flags"] && cp.columns.map(\.physical) == [1, 5, 6, 1])266        #expect(try (0..<3).map { try cp.read(column: 2, group: 0).values.text($0) } == ["p1", "proton", ""])267    }268269    @Test func severalRowGroups() async throws {270        // 3 000 000 rows: more than one block, so more than one row group; every value is checked.271        let temp = TempDir()272        let url = temp.file("big.h5")273        let n = 3_000_000274        H5.sync {275            let f = C.create(url)276            C.numbers(f, "v", ORIX_T_INT32, (0..<n).map { Int32(truncatingIfNeeded: $0 &* 7) })277            H5Fclose(f)278        }279        let r = try await convert(url, .hdf5, to: .parquet, in: temp)280        let p = try ParquetFile(url: r.file)281        #expect(p.rows == Int64(n) && p.rowGroupCount == 3)282        var index = 0283        for g in 0..<p.rowGroupCount {284            guard case .int(let a) = try p.read(column: 0, group: g).values else { Issue.record("not integers"); return }285            for v in a { if v != Int64(Int32(truncatingIfNeeded: index &* 7)) { Issue.record("row \(index) = \(v)"); return }; index += 1 }286        }287        #expect(index == n)288        #expect(r.progress.fractions == r.progress.fractions.sorted() && r.progress.fractions.count >= 3)289    }290291    @Test func parquetReaderRejectsGarbage() throws {292        let temp = TempDir()293        for (name, bytes) in [("short", Data("PAR1".utf8)), ("magic", Data(count: 64)), ("footer", Data("PAR1".utf8) + Data([0xFF, 0xFF, 0xFF, 0x7F]) + Data("PAR1".utf8) + Data(count: 4))] {294            let url = temp.file(name + ".parquet")295            try bytes.write(to: url)296            #expect(throws: ConversionError.self) { _ = try ParquetFile(url: url) }297        }298        // A footer of random bytes with a plausible length.299        var data = Data("PAR1".utf8)300        data.append(Data((0..<200).map { UInt8(truncatingIfNeeded: $0 &* 37 &+ 11) }))301        data.append(contentsOf: [200, 0, 0, 0])302        data.append(Data("PAR1".utf8))303        let url = temp.file("random.parquet")304        try data.write(to: url)305        do { _ = try ParquetFile(url: url) } catch is ConversionError {} catch { Issue.record("unexpected error \(error)") }306    }307}308
Files main
Menu

Theme