Repository Public License
1// ORIX Convert — Packages/EngineScientific/Tests/EngineScientificTests/TranscodeTests.swift2// Description: Structure-preserving conversions between HDF5, NetCDF-4 and NetCDF classic: every value, shape,3// dimension, attribute, chunk and filter setting is compared; what a destination cannot hold is checked4// to be refused or reported, exactly.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import Testing8import CHDF59import CHDF5Shim10import ConvertCore11@testable import EngineScientific1213/// Every value of a dataset as text, read block by block.14func allValues(_ file: SciFile, _ path: String) throws -> [String] {15 let info = try #require(try file.node(at: path).dataset)16 var out: [String] = []17 for block in BlockSequence(SciSelection(all: info.shape), maxElements: 50_000) {18 let v = try file.backend.read(path, block)19 if case .unsupported = v { return ["<unsupported>"] }20 if case .compound(let members) = v { out += (0..<v.count).map { i in members.map { $0.values.text(i) }.joined(separator: "|") } } else { out += (0..<v.count).map { v.text($0) } }21 }22 return out23}2425/// Compares datasets (shape, type, unlimited flags, every value) and attributes (names, types, values).26func expectSame(_ a: SciFile, _ b: SciFile, paths: [String], types: Bool = true, sourceLocation: SourceLocation = #_sourceLocation) throws {27 for path in paths {28 let x = try #require(try a.node(at: path).dataset, sourceLocation: sourceLocation), y = try #require(try b.node(at: path).dataset, sourceLocation: sourceLocation)29 #expect(x.shape == y.shape, "\(path): shape", sourceLocation: sourceLocation)30 if types { #expect(x.type == y.type, "\(path): type \(x.type.description) vs \(y.type.description)", sourceLocation: sourceLocation) }31 #expect(x.dimensions.map(\.isUnlimited) == y.dimensions.map(\.isUnlimited), "\(path): unlimited dimensions", sourceLocation: sourceLocation)32 #expect(try allValues(a, path) == allValues(b, path), "\(path): values", sourceLocation: sourceLocation)33 let p = try a.attributes(of: path, maxElements: 10_000).items, q = try b.attributes(of: path, maxElements: 10_000).items34 #expect(p.map(\.name).sorted() == q.map(\.name).sorted(), "\(path): attribute names", sourceLocation: sourceLocation)35 for attribute in p {36 guard let other = q.first(where: { $0.name == attribute.name }) else { continue }37 #expect(attribute.text(maxElements: 10_000) == other.text(maxElements: 10_000), "\(path)@\(attribute.name)", sourceLocation: sourceLocation)38 if types { #expect(attribute.type == other.type, "\(path)@\(attribute.name): type", sourceLocation: sourceLocation) }39 }40 }41}4243func datasetPaths(_ file: SciFile) throws -> [String] { try file.tree().nodes.filter { $0.kind == .dataset }.map(\.path) }4445@Suite struct HDF5ToNetCDFTests {46 @Test func everythingToNetCDF4() async throws {47 let temp = TempDir()48 let sourceURL = Fixture.everything(in: temp.url)49 let r = try await convert(sourceURL, .hdf5, to: .netcdf, in: temp)50 #expect(r.file.pathExtension == "nc" && FileTypeDetector.detect(r.file).format == .netcdf)51 let source = try SciFile(url: sourceURL), out = try SciFile(url: r.file)52 #expect(out.formatDescription == "NetCDF-4 (HDF5)")53 // What NetCDF cannot hold is left out and named, one warning each.54 let skipped = r.warnings.all("sci-dataset-skipped").map(\.message).sorted()55 #expect(skipped == [56 "The dataset /array/vectors is not in the output: it holds an array datatype, which a NetCDF variable cannot have.",57 "The dataset /compound/particles is not in the output: it holds a compound type with text members, which NetCDF cannot represent.",58 "The dataset /floats/f16 is not in the output: it holds 16-bit floating-point numbers, which NetCDF does not have.",59 "The dataset /refs/objref is not in the output: it holds object or region references.",60 ])61 #expect(r.warnings.all("sci-dataset-skipped").allSatisfy { $0.severity == .dataLoss })62 #expect(r.warnings.first("sci-external-link")?.message == "/links/ext is a link to /data in another file (other.h5). External links are never followed: the object is not in the output.")63 #expect(r.warnings.first("sci-dangling-link")?.message.contains("/links/dangling is a soft link to /nowhere") == true)64 #expect(r.warnings.first("sci-soft-link")?.message.hasPrefix("/links/soft is a soft link to /scalar. NetCDF has no links") == true)65 #expect(r.warnings.first("sci-dataset-alias")?.message.hasPrefix("/links/alias is a second name of the dataset /ints/i32") == true)66 // The grid's scales live in a sibling group, which NetCDF scoping does not allow: said, twice (two dimensions).67 #expect(r.warnings.all("sci-scale-scope").count == 2)68 let paths = try datasetPaths(out)69 #expect(paths.count == 22 && r.output.facts["sci.datasets"] == "22" && r.output.facts["sci.skipped"] == "4" && r.output.facts["sci.groups"] == "13")70 #expect(!paths.contains("/links/ext") && !paths.contains("/links/soft") && !paths.contains("/links/alias"))71 try expectSame(source, out, paths: paths)72 #expect(try out.attributes(of: "/").items.map(\.name) == ["ratios", "tags", "title", "version"])73 #expect(try out.attributes(of: "/ints").items.first?.text() == "integers")74 // Storage: chunking, filters, fill value, unlimited dimension.75 let grid = try #require(try out.node(at: "/chunked/grid").dataset)76 #expect(grid.chunkShape == [5, 10] && grid.filters.map(\.name) == ["shuffle", "deflate"] && grid.filters[1].parameters.first == 6 && grid.fillValue == "-999")77 #expect(grid.dimensions.allSatisfy { $0.name?.hasPrefix("phony_dim_") == true && $0.coordinatePath == nil } && grid.dimensions[0].name != grid.dimensions[1].name)78 // The eight integer datasets share one generated dimension of length 4, defined in their group.79 let integerDimensions = try Set(["i8", "u8", "i16", "u16", "i32", "u32", "i64", "u64"].map { try out.node(at: "/ints/\($0)").dataset?.dimensions.first?.definitionPath ?? "" })80 #expect(integerDimensions.count == 1 && integerDimensions.first?.hasPrefix("/ints/phony_dim_") == true)81 let placeholderPath = try #require(integerDimensions.first)82 #expect(try out.node(at: "/unlimited/series").dataset?.dimensions.first?.isUnlimited == true)83 #expect(try out.node(at: "/unlimited/series").dataset?.chunkShape == [4])84 // A scale in the same group stays a named dimension with its coordinate variable.85 #expect(try out.node(at: "/time/values").dataset?.dimensions == [SciDimension(name: "t", length: 5, isUnlimited: false, coordinatePath: "/time/t")])86 // The NetCDF-4 machinery, checked with the C library directly.87 try H5.sync {88 let fapl = orix_h5_fapl(0)89 defer { H5Pclose(fapl) }90 let f = H5Fopen(r.file.path, UInt32(H5F_ACC_RDONLY), fapl)91 try #require(f >= 0)92 defer { H5Fclose(f) }93 #expect(H5Aexists(f, "_NCProperties") > 0)94 let t = H5Dopen2(f, "/time/t", 0), placeholder = H5Dopen2(f, placeholderPath, 0), i8 = H5Dopen2(f, "/ints/i8", 0)95 defer { H5Dclose(t); H5Dclose(placeholder); H5Dclose(i8) }96 #expect(H5DSis_scale(t) > 0 && H5DSis_scale(placeholder) > 0 && H5DSis_scale(i8) == 0)97 #expect(H5Aexists(t, "_Netcdf4Dimid") > 0 && H5Aexists(placeholder, "_Netcdf4Dimid") > 0)98 #expect(H5DSis_attached(i8, placeholder, 0) > 0)99 var name = [CChar](repeating: 0, count: 128)100 H5DSget_scale_name(placeholder, &name, 128)101 #expect(H5.text(name) == "This is a netCDF dimension but not a netCDF variable. 4")102 for group in ["/", "/ints"] {103 let g = H5Gopen2(f, group, 0)104 let gcpl = H5Gget_create_plist(g)105 var flags: UInt32 = 0106 H5Pget_link_creation_order(gcpl, &flags)107 H5Pclose(gcpl); H5Gclose(g)108 #expect(flags & UInt32(H5P_CRT_ORDER_TRACKED) != 0, "link creation order of \(group)")109 }110 }111 // The dimension placeholders are dimensions, not data: hidden when the file is read back.112 #expect(try out.children(of: "/ints").map(\.name) == ["i16", "i32", "i64", "i8", "u16", "u32", "u64", "u8"])113 }114115 @Test func classicVariantsFromHDF5() async throws {116 let temp = TempDir()117 let single = Fixture.single(temp.file("single.h5"))118 for (variant, version, description) in [("classic", UInt8(1), "NetCDF classic (CDF-1)"), ("64bit-offset", 2, "NetCDF classic (CDF-2, 64-bit offset)"), ("cdf5", 5, "NetCDF classic (CDF-5, 64-bit data)"), ("classic-auto", 1, "NetCDF classic (CDF-1)")] {119 let r = try await convert(single, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string(variant)])120 #expect([UInt8](try Data(contentsOf: r.file).prefix(4)) == [0x43, 0x44, 0x46, version])121 let out = try SciFile(url: r.file)122 #expect(out.formatDescription == description && r.output.facts["sci.format"] == description)123 #expect(try out.node(at: "/m").dataset?.dimensions.map(\.name) == ["dim_3", "dim_4"])124 #expect(try allValues(out, "/m") == (0..<12).map(String.init))125 }126 // The exact bytes of the CDF-1 file, as the specification lays them out.127 let r = try await convert(single, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic")])128 var expected: [UInt8] = [0x43, 0x44, 0x46, 1, 0, 0, 0, 0, 0, 0, 0, 0x0A, 0, 0, 0, 2]129 expected += [0, 0, 0, 5] + Array("dim_3".utf8) + [0, 0, 0] + [0, 0, 0, 3] + [0, 0, 0, 5] + Array("dim_4".utf8) + [0, 0, 0] + [0, 0, 0, 4]130 expected += [0, 0, 0, 0, 0, 0, 0, 0] // no global attributes131 expected += [0, 0, 0, 0x0B, 0, 0, 0, 1, 0, 0, 0, 1] + Array("m".utf8) + [0, 0, 0]132 expected += [0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1] // two dimensions: 0, 1133 expected += [0, 0, 0, 0, 0, 0, 0, 0] // no attributes134 expected += [0, 0, 0, 4, 0, 0, 0, 48, 0, 0, 0, 104] // NC_INT, vsize 48, begin 104135 for v in 0..<12 { expected += [0, 0, 0, UInt8(v)] }136 #expect([UInt8](try Data(contentsOf: r.file)) == expected)137 }138139 @Test func classicRefusals() async throws {140 let temp = TempDir()141 let everything = Fixture.everything(in: temp.url)142 let grouped = await failure(everything, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic-auto")])143 #expect(grouped.hasPrefix("A setting is not valid. NetCDF classic has no groups, and 26 dataset(s) live inside groups (/array/vectors, /chunked/grid, /compound/particles, /dims/x, /dims/y…). Choose NetCDF-4 as the destination format."), "\(grouped)")144 // Root-level datasets of types classic cannot hold.145 let url = temp.file("types.h5")146 H5.sync {147 let f = C.create(url)148 let vs = C.variableString()149 let words: [UnsafeMutablePointer<CChar>?] = [strdup("a"), strdup("b")]150 words.withUnsafeBytes { _ = C.dataset(f, "words", file: vs, memory: vs, dims: [2], data: $0.baseAddress) }151 words.forEach { free($0) }152 H5Tclose(vs)153 C.numbers(f, "big", ORIX_T_UINT64, [UInt64.max])154 C.numbers(f, "ok", ORIX_T_INT16, [Int16(1), 2])155 H5Fclose(f)156 }157 let strings = await failure(url, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic-auto")])158 #expect(strings == "A setting is not valid. NetCDF classic only holds numbers and characters. 1 variable(s) cannot be stored: /words (string). Choose NetCDF-4 as the destination format.")159 // Unsigned 64-bit integers need CDF-5: an explicit older variant is refused, the automatic choice picks it.160 let numbers = temp.file("numbers.h5")161 H5.sync {162 let f = C.create(numbers)163 C.numbers(f, "big", ORIX_T_UINT64, [UInt64.max, 7])164 C.attribute(f, "names", ORIX_T_INT8, [Int8(1)])165 H5Fclose(f)166 }167 #expect(await failure(numbers, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic")]).contains("The content needs the CDF-5 variant of NetCDF classic"))168 #expect(await failure(numbers, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("64bit-offset")]).contains("The content needs the CDF-5 variant"))169 let auto = try await convert(numbers, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic-auto")])170 #expect(auto.output.facts["sci.format"] == "NetCDF classic (CDF-5, 64-bit data)")171 #expect(try allValues(SciFile(url: auto.file), "/big") == ["18446744073709551615", "7"])172 #expect(await failure(numbers, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("zarr")]).contains("is not a NetCDF variant"))173 }174}175176@Suite struct NetCDFConversionTests {177 @Test(arguments: [UInt8(1), 2, 5]) func classicWriterReproducesIndependentBytes(_ version: UInt8) async throws {178 // The fixture is assembled byte by byte in the test; rewriting it in the same variant must give the same file.179 let temp = TempDir()180 let source = Fixture.classic(temp.file("c.nc"), version: version)181 let variant = version == 1 ? "classic" : version == 2 ? "64bit-offset" : "cdf5"182 let r = try await convert(source, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, options: ["netcdf.format": .string(variant)])183 #expect(try Data(contentsOf: r.file) == Data(contentsOf: source))184 #expect(r.warnings.all.filter { $0.severity == .dataLoss }.isEmpty)185 }186187 @Test func classicToNetCDF4AndBack() async throws {188 let temp = TempDir()189 let sourceURL = Fixture.classic(temp.file("c.nc"), version: 5)190 let source = try SciFile(url: sourceURL)191 // "Automatic" means NetCDF-4 for a classic source.192 let nc4 = try await convert(sourceURL, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, options: ["compression": .string("deflate"), "compression.level": .int(7)], name: "as4")193 let out = try SciFile(url: nc4.file)194 #expect(out.formatDescription == "NetCDF-4 (HDF5)" && nc4.output.facts["sci.dimensions"] == "4" && nc4.output.facts["sci.datasets"] == "9")195 let paths = try datasetPaths(out)196 #expect(paths == ["/lat", "/lon", "/station", "/time", "/temp", "/scalarv", "/pres", "/big", "/flags"])197 // Characters are a one-character string type in NetCDF-4; everything else keeps its exact type.198 try expectSame(source, out, paths: paths.filter { $0 != "/station" })199 #expect(try out.node(at: "/station").dataset?.type == .string(fixedLength: 1))200 #expect(try allValues(out, "/station") == ["A", "B", "C", "D", "E", "F", "", ""])201 #expect(try out.attributes(of: "/").items.map { "\($0.name)=\($0.text())" } == ["title=Classic", "version=7"])202 let t = try #require(try out.node(at: "/temp").dataset)203 #expect(t.dimensions == [SciDimension(name: "time", length: 3, isUnlimited: true, coordinatePath: "/time"), SciDimension(name: "lat", length: 2, isUnlimited: false, coordinatePath: "/lat"),204 SciDimension(name: "lon", length: 3, isUnlimited: false, coordinatePath: "/lon")])205 #expect(t.fillValue == "-999" && t.filters.map(\.name) == ["shuffle", "deflate"] && t.filters[1].parameters.first == 7)206 // "strlen" has no coordinate variable: a dimension placeholder, not a dataset.207 #expect(try out.node(at: "/station").dataset?.dimensions[1] == SciDimension(name: "strlen", length: 4, isUnlimited: false, coordinatePath: nil, definitionPath: "/strlen"))208 // And back: automatic means classic for a NetCDF-4 source, the smallest variant that fits (CDF-5 here).209 let back = try await convert(nc4.file, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, name: "back")210 let again = try SciFile(url: back.file)211 #expect(again.formatDescription == "NetCDF classic (CDF-5, 64-bit data)")212 try expectSame(source, again, paths: paths)213 #expect(back.warnings.first("nc-storage-dropped") != nil)214 // To HDF5: the same content, still carrying the NetCDF-4 dimension scales.215 let h5 = try await convert(sourceURL, .netcdf, to: .hdf5, in: temp)216 #expect(h5.file.pathExtension == "h5" && FileTypeDetector.detect(h5.file).format == .hdf5)217 try expectSame(source, try SciFile(url: h5.file), paths: paths.filter { $0 != "/station" })218 }219220 @Test func netCDF4ToHDF5AndRecompression() async throws {221 let temp = TempDir()222 let sourceURL = Fixture.netCDF4(temp.file("raw.nc"))223 let source = try SciFile(url: sourceURL)224 let paths = ["/x", "/temp", "/label", "/model/bias"]225 let h5 = try await convert(sourceURL, .netcdf, to: .hdf5, in: temp)226 let out = try SciFile(url: h5.file)227 try expectSame(source, out, paths: paths)228 let names = try out.children().map(\.name)229 #expect(out.isNetCDF && names == ["x", "temp", "label", "model"])230 let sourceDimensions = try source.node(at: "/temp").dataset?.dimensions231 #expect(try out.node(at: "/temp").dataset?.dimensions == sourceDimensions)232 #expect(try out.node(at: "/temp").dataset?.filters.map(\.name) == ["deflate"] && out.node(at: "/temp").dataset?.chunkShape == [1, 3])233 #expect(try out.node(at: "/model/bias").dataset?.dimensions.first?.coordinatePath == "/x")234 let none = try await convert(sourceURL, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, options: ["netcdf.format": .string("netcdf4"), "compression": .string("none")])235 #expect(try SciFile(url: none.file).node(at: "/temp").dataset?.filters.isEmpty == true)236 // An unlimited dimension keeps its chunking even without compression.237 #expect(try SciFile(url: none.file).node(at: "/temp").dataset?.chunkShape == [1, 3])238 let deflate = try await convert(sourceURL, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, options: ["netcdf.format": .string("netcdf4"), "compression": .string("deflate"), "compression.level": .int(9), "compression.shuffle": .bool(false)])239 let d = try SciFile(url: deflate.file)240 #expect(try d.node(at: "/x").dataset?.filters.map(\.name) == ["deflate"] && d.node(at: "/x").dataset?.filters.first?.parameters.first == 9)241 try expectSame(source, d, paths: paths)242 // NetCDF-4 with a group cannot become classic.243 #expect(await failure(sourceURL, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation).contains("NetCDF classic has no groups, and 1 dataset(s) live inside groups (/model/bias)"))244 }245246 @Test func severalUnlimitedDimensionsToClassic() async throws {247 let temp = TempDir()248 let url = temp.file("two.h5")249 H5.sync {250 let f = C.create(url)251 let dcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))252 H5Pset_chunk(dcpl, 2, [hsize_t(2), 2])253 C.numbers(f, "a", ORIX_T_INT32, [Int32(1), 2, 3, 4, 5, 6], dims: [2, 3], maxDims: [H5.unlimited, H5.unlimited], dcpl: dcpl)254 H5Pset_chunk(dcpl, 1, [hsize_t(2)])255 C.numbers(f, "b", ORIX_T_FLOAT, [Float(1.5), 2.5], dims: [2], maxDims: [H5.unlimited], dcpl: dcpl)256 H5Pclose(dcpl)257 C.attribute(f, "weird name/with slash", text: "x")258 let vs = C.variableString()259 let two: [UnsafeMutablePointer<CChar>?] = [strdup("one"), strdup("two")]260 two.withUnsafeBytes { C.attribute(f, "lines", file: vs, memory: vs, dims: [2], data: $0.baseAddress!) }261 two.forEach { free($0) }262 H5Tclose(vs)263 H5Fclose(f)264 }265 let r = try await convert(url, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic")])266 let out = try SciFile(url: r.file)267 // "a" and "b" share the generated dimension of length 2, which leads both: it is the record dimension.268 #expect(try out.node(at: "/a").dataset?.dimensions.map { "\($0.name ?? "")=\($0.length)\($0.isUnlimited ? "u" : "")" } == ["dim_2=2u", "dim_3=3"])269 #expect(try out.node(at: "/b").dataset?.dimensions.map { "\($0.name ?? "")=\($0.length)\($0.isUnlimited ? "u" : "")" } == ["dim_2=2u"])270 #expect(r.warnings.first("nc-unlimited-fixed")?.message == "The unlimited dimension dim_3 is written with its current fixed length 3: NetCDF classic allows one unlimited dimension, used first.")271 #expect(try allValues(out, "/a") == ["1", "2", "3", "4", "5", "6"] && allValues(out, "/b") == ["1.5", "2.5"])272 #expect(r.warnings.first("nc-string-attributes-joined")?.message.hasPrefix("1 attribute(s)") == true)273 #expect(r.warnings.first("nc-storage-dropped") != nil)274 #expect(try out.attributes(of: "/").items.map { "\($0.name)=\($0.text())" } == ["lines=one\ntwo", "weird name_with slash=x"])275 }276}277278@Suite struct RepackTests {279 @Test func repackKeepsEverythingItCan() async throws {280 let temp = TempDir()281 let sourceURL = Fixture.everything(in: temp.url)282 let source = try SciFile(url: sourceURL)283 let r = try await convert(sourceURL, .hdf5, to: .hdf5, in: temp, operation: ScientificEngine.repackOperation, options: ["compression": .string("deflate"), "compression.level": .int(5)])284 let out = try SciFile(url: r.file)285 #expect(out.formatDescription == "HDF5" && !out.isNetCDF)286 let paths = try datasetPaths(out)287 // 28 names in the source: the reference dataset is left out; the hard link and the soft link are links again.288 #expect(paths.count == 27 && !paths.contains("/refs/objref") && r.output.facts["sci.datasets"] == "25" && r.output.facts["sci.links"] == "2")289 try expectSame(source, out, paths: paths)290 #expect(r.warnings.all("sci-dataset-skipped").map(\.message) == ["The dataset /refs/objref is not in the output: it holds object or region references."])291 #expect(r.warnings.first("sci-external-link") != nil && r.warnings.first("sci-dangling-link") != nil)292 #expect(try out.children(of: "/links").map { "\($0.name):\($0.softLinkTarget ?? "-")" } == ["alias:-", "soft:/scalar"])293 #expect(out.backend.identity(of: "/links/alias") == out.backend.identity(of: "/ints/i32"))294 // Types that only HDF5 has survive: compound with text, array, enumeration, half floats.295 #expect(try out.node(at: "/compound/particles").dataset?.type.description == "compound {id: int32, mass: float64, name: string(6), flags: uint8}")296 #expect(try allValues(out, "/compound/particles") == ["1|1.5|p1|1", "2|2.25|proton|200", "-3|0.0||0"])297 #expect(try allValues(out, "/floats/f16") == ["0.5", "-2.0"] && allValues(out, "/enum/colors") == ["RED", "BLUE", "GREEN", "BLUE"])298 // Dimension scales are attached again; storage settings follow the options.299 let grid = try #require(try out.node(at: "/chunked/grid").dataset)300 #expect(grid.dimensions == [SciDimension(name: "y", length: 20, isUnlimited: false, coordinatePath: "/dims/y"), SciDimension(name: "x", length: 30, isUnlimited: false, coordinatePath: "/dims/x")])301 #expect(grid.chunkShape == [5, 10] && grid.filters.map(\.name) == ["shuffle", "deflate"] && grid.filters[1].parameters.first == 5 && grid.fillValue == "-999")302 #expect(try out.node(at: "/ints/i64").dataset?.filters.map(\.name) == ["shuffle", "deflate"])303 #expect(try out.node(at: "/ints/i8").dataset?.filters.map(\.name) == ["deflate"], "one-byte elements are not shuffled")304 #expect(try out.node(at: "/scalar").dataset?.filters.isEmpty == true, "a scalar cannot be chunked")305 #expect(try out.node(at: "/unlimited/series").dataset?.dimensions.first?.isUnlimited == true)306 #expect(try out.node(at: "/floats/f64be").dataset?.type == .float(bits: 64))307 // "Same as the source" and "none".308 let same = try SciFile(url: try await convert(sourceURL, .hdf5, to: .hdf5, in: temp, operation: ScientificEngine.repackOperation).file)309 #expect(try same.node(at: "/chunked/grid").dataset?.filters[1].parameters.first == 6 && same.node(at: "/ints/i64").dataset?.layout == .contiguous)310 let none = try SciFile(url: try await convert(sourceURL, .hdf5, to: .hdf5, in: temp, operation: ScientificEngine.repackOperation, options: ["compression": .string("none")]).file)311 #expect(try none.node(at: "/chunked/grid").dataset?.filters.isEmpty == true && none.node(at: "/unlimited/series").dataset?.chunkShape == [4])312 try expectSame(source, none, paths: paths)313 }314315 @Test func storageThatIsNeverFollowed() async throws {316 let temp = TempDir()317 let url = temp.file("storage.h5")318 try Data([1, 0, 0, 0, 2, 0, 0, 0]).write(to: temp.file("raw.bin"))319 H5.sync {320 let f = C.create(url)321 // Raw data in an external file.322 let dcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))323 H5Pset_external(dcpl, "raw.bin", 0, 8)324 C.dataset(f, "external", file: H5.type(ORIX_T_INT32), memory: H5.type(ORIX_T_INT32), dims: [2], dcpl: dcpl, data: nil)325 H5Pclose(dcpl)326 // A virtual dataset mapped onto another file.327 let vcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))328 var n = hsize_t(2)329 let vspace = H5Screate_simple(1, &n, nil), sspace = H5Screate_simple(1, &n, nil)330 H5Pset_virtual(vcpl, vspace, "other.h5", "/data", sspace)331 C.dataset(f, "virtual", file: H5.type(ORIX_T_INT32), memory: H5.type(ORIX_T_INT32), dims: [2], dcpl: vcpl, data: nil)332 H5Sclose(vspace); H5Sclose(sspace); H5Pclose(vcpl)333 C.numbers(f, "plain", ORIX_T_INT32, [Int32(5), 6])334 H5Fclose(f)335 let other = C.create(temp.file("other.h5"))336 C.numbers(other, "data", ORIX_T_INT32, [Int32(111), 222])337 H5Fclose(other)338 }339 let file = try SciFile(url: url)340 let external = try #require(try file.node(at: "/external").dataset), virtual = try #require(try file.node(at: "/virtual").dataset)341 #expect(external.layout == .external && external.unreadableReason == "its raw data is stored in external files (never followed)")342 #expect(virtual.layout == .virtual && virtual.unreadableReason == "it is a virtual dataset, whose data may live in other files (never followed)")343 #expect(throws: ConversionError.malformedInput("/external cannot be read: its raw data is stored in external files (never followed).")) { _ = try file.backend.read("/external", SciSelection(all: [2])) }344 #expect(throws: ConversionError.self) { _ = try file.backend.read("/virtual", SciSelection(all: [2])) }345 #expect(try file.preview("/external").note == "The values cannot be read: its raw data is stored in external files (never followed).")346 #expect(await failure(url, .hdf5, to: .csv, in: temp, options: ["variables": .string("/virtual")]).contains("/virtual cannot be read: it is a virtual dataset"))347 let r = try await convert(url, .hdf5, to: .netcdf, in: temp)348 #expect(r.warnings.all("sci-dataset-skipped").map(\.message).sorted() == [349 "The dataset /external is not in the output: its raw data is stored in external files (never followed).",350 "The dataset /virtual is not in the output: it is a virtual dataset, whose data may live in other files (never followed).",351 ])352 #expect(try datasetPaths(SciFile(url: r.file)) == ["/plain"])353 let i = try await ScientificEngine().inspect(url, format: .hdf5)354 #expect(i.fields.first { $0.key == "/external" }?.value.contains("unreadable: its raw data is stored in external files") == true)355 }356357 @Test func cancellationAndProgress() async throws {358 let temp = TempDir()359 let url = temp.file("large.h5")360 H5.sync {361 let f = C.create(url)362 C.numbers(f, "v", ORIX_T_DOUBLE, (0..<4_000_000).map { Double($0) })363 H5Fclose(f)364 }365 let r = try await convert(url, .hdf5, to: .hdf5, in: temp, operation: ScientificEngine.repackOperation, options: ["compression": .string("deflate"), "compression.level": .int(1)])366 #expect(r.progress.fractions.count >= 3 && r.progress.fractions == r.progress.fractions.sorted() && r.progress.fractions.last == 1)367 #expect(r.output.facts["sci.elements"] == "4000000")368 for destination in [FormatID.hdf5, .csv, .parquet] {369 let work = temp.sub("cancel-\(destination.rawValue)")370 let task = Task {371 try await ScientificEngine().convert(ConversionRequest(source: url, sourceFormat: .hdf5, destinationFormat: destination, operation: destination == .hdf5 ? ScientificEngine.repackOperation : nil),372 context: ConversionContext(workDirectory: work))373 }374 task.cancel()375 await #expect(throws: CancellationError.self) { _ = try await task.value }376 }377 }378}379