SPB Git

Repository Public License

TranscodeTests.swift 29.0 KB 379 lines swift

1// ORIX Convert — Packages/EngineScientific/Tests/EngineScientificTests/TranscodeTests.swift2// Description: Structure-preserving conversions between HDF5, NetCDF-4 and NetCDF classic: every value, shape,3//              dimension, attribute, chunk and filter setting is compared; what a destination cannot hold is checked4//              to be refused or reported, exactly.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import Testing8import CHDF59import CHDF5Shim10import ConvertCore11@testable import EngineScientific1213/// Every value of a dataset as text, read block by block.14func allValues(_ file: SciFile, _ path: String) throws -> [String] {15    let info = try #require(try file.node(at: path).dataset)16    var out: [String] = []17    for block in BlockSequence(SciSelection(all: info.shape), maxElements: 50_000) {18        let v = try file.backend.read(path, block)19        if case .unsupported = v { return ["<unsupported>"] }20        if case .compound(let members) = v { out += (0..<v.count).map { i in members.map { $0.values.text(i) }.joined(separator: "|") } } else { out += (0..<v.count).map { v.text($0) } }21    }22    return out23}2425/// Compares datasets (shape, type, unlimited flags, every value) and attributes (names, types, values).26func expectSame(_ a: SciFile, _ b: SciFile, paths: [String], types: Bool = true, sourceLocation: SourceLocation = #_sourceLocation) throws {27    for path in paths {28        let x = try #require(try a.node(at: path).dataset, sourceLocation: sourceLocation), y = try #require(try b.node(at: path).dataset, sourceLocation: sourceLocation)29        #expect(x.shape == y.shape, "\(path): shape", sourceLocation: sourceLocation)30        if types { #expect(x.type == y.type, "\(path): type \(x.type.description) vs \(y.type.description)", sourceLocation: sourceLocation) }31        #expect(x.dimensions.map(\.isUnlimited) == y.dimensions.map(\.isUnlimited), "\(path): unlimited dimensions", sourceLocation: sourceLocation)32        #expect(try allValues(a, path) == allValues(b, path), "\(path): values", sourceLocation: sourceLocation)33        let p = try a.attributes(of: path, maxElements: 10_000).items, q = try b.attributes(of: path, maxElements: 10_000).items34        #expect(p.map(\.name).sorted() == q.map(\.name).sorted(), "\(path): attribute names", sourceLocation: sourceLocation)35        for attribute in p {36            guard let other = q.first(where: { $0.name == attribute.name }) else { continue }37            #expect(attribute.text(maxElements: 10_000) == other.text(maxElements: 10_000), "\(path)@\(attribute.name)", sourceLocation: sourceLocation)38            if types { #expect(attribute.type == other.type, "\(path)@\(attribute.name): type", sourceLocation: sourceLocation) }39        }40    }41}4243func datasetPaths(_ file: SciFile) throws -> [String] { try file.tree().nodes.filter { $0.kind == .dataset }.map(\.path) }4445@Suite struct HDF5ToNetCDFTests {46    @Test func everythingToNetCDF4() async throws {47        let temp = TempDir()48        let sourceURL = Fixture.everything(in: temp.url)49        let r = try await convert(sourceURL, .hdf5, to: .netcdf, in: temp)50        #expect(r.file.pathExtension == "nc" && FileTypeDetector.detect(r.file).format == .netcdf)51        let source = try SciFile(url: sourceURL), out = try SciFile(url: r.file)52        #expect(out.formatDescription == "NetCDF-4 (HDF5)")53        // What NetCDF cannot hold is left out and named, one warning each.54        let skipped = r.warnings.all("sci-dataset-skipped").map(\.message).sorted()55        #expect(skipped == [56            "The dataset /array/vectors is not in the output: it holds an array datatype, which a NetCDF variable cannot have.",57            "The dataset /compound/particles is not in the output: it holds a compound type with text members, which NetCDF cannot represent.",58            "The dataset /floats/f16 is not in the output: it holds 16-bit floating-point numbers, which NetCDF does not have.",59            "The dataset /refs/objref is not in the output: it holds object or region references.",60        ])61        #expect(r.warnings.all("sci-dataset-skipped").allSatisfy { $0.severity == .dataLoss })62        #expect(r.warnings.first("sci-external-link")?.message == "/links/ext is a link to /data in another file (other.h5). External links are never followed: the object is not in the output.")63        #expect(r.warnings.first("sci-dangling-link")?.message.contains("/links/dangling is a soft link to /nowhere") == true)64        #expect(r.warnings.first("sci-soft-link")?.message.hasPrefix("/links/soft is a soft link to /scalar. NetCDF has no links") == true)65        #expect(r.warnings.first("sci-dataset-alias")?.message.hasPrefix("/links/alias is a second name of the dataset /ints/i32") == true)66        // The grid's scales live in a sibling group, which NetCDF scoping does not allow: said, twice (two dimensions).67        #expect(r.warnings.all("sci-scale-scope").count == 2)68        let paths = try datasetPaths(out)69        #expect(paths.count == 22 && r.output.facts["sci.datasets"] == "22" && r.output.facts["sci.skipped"] == "4" && r.output.facts["sci.groups"] == "13")70        #expect(!paths.contains("/links/ext") && !paths.contains("/links/soft") && !paths.contains("/links/alias"))71        try expectSame(source, out, paths: paths)72        #expect(try out.attributes(of: "/").items.map(\.name) == ["ratios", "tags", "title", "version"])73        #expect(try out.attributes(of: "/ints").items.first?.text() == "integers")74        // Storage: chunking, filters, fill value, unlimited dimension.75        let grid = try #require(try out.node(at: "/chunked/grid").dataset)76        #expect(grid.chunkShape == [5, 10] && grid.filters.map(\.name) == ["shuffle", "deflate"] && grid.filters[1].parameters.first == 6 && grid.fillValue == "-999")77        #expect(grid.dimensions.allSatisfy { $0.name?.hasPrefix("phony_dim_") == true && $0.coordinatePath == nil } && grid.dimensions[0].name != grid.dimensions[1].name)78        // The eight integer datasets share one generated dimension of length 4, defined in their group.79        let integerDimensions = try Set(["i8", "u8", "i16", "u16", "i32", "u32", "i64", "u64"].map { try out.node(at: "/ints/\($0)").dataset?.dimensions.first?.definitionPath ?? "" })80        #expect(integerDimensions.count == 1 && integerDimensions.first?.hasPrefix("/ints/phony_dim_") == true)81        let placeholderPath = try #require(integerDimensions.first)82        #expect(try out.node(at: "/unlimited/series").dataset?.dimensions.first?.isUnlimited == true)83        #expect(try out.node(at: "/unlimited/series").dataset?.chunkShape == [4])84        // A scale in the same group stays a named dimension with its coordinate variable.85        #expect(try out.node(at: "/time/values").dataset?.dimensions == [SciDimension(name: "t", length: 5, isUnlimited: false, coordinatePath: "/time/t")])86        // The NetCDF-4 machinery, checked with the C library directly.87        try H5.sync {88            let fapl = orix_h5_fapl(0)89            defer { H5Pclose(fapl) }90            let f = H5Fopen(r.file.path, UInt32(H5F_ACC_RDONLY), fapl)91            try #require(f >= 0)92            defer { H5Fclose(f) }93            #expect(H5Aexists(f, "_NCProperties") > 0)94            let t = H5Dopen2(f, "/time/t", 0), placeholder = H5Dopen2(f, placeholderPath, 0), i8 = H5Dopen2(f, "/ints/i8", 0)95            defer { H5Dclose(t); H5Dclose(placeholder); H5Dclose(i8) }96            #expect(H5DSis_scale(t) > 0 && H5DSis_scale(placeholder) > 0 && H5DSis_scale(i8) == 0)97            #expect(H5Aexists(t, "_Netcdf4Dimid") > 0 && H5Aexists(placeholder, "_Netcdf4Dimid") > 0)98            #expect(H5DSis_attached(i8, placeholder, 0) > 0)99            var name = [CChar](repeating: 0, count: 128)100            H5DSget_scale_name(placeholder, &name, 128)101            #expect(H5.text(name) == "This is a netCDF dimension but not a netCDF variable.         4")102            for group in ["/", "/ints"] {103                let g = H5Gopen2(f, group, 0)104                let gcpl = H5Gget_create_plist(g)105                var flags: UInt32 = 0106                H5Pget_link_creation_order(gcpl, &flags)107                H5Pclose(gcpl); H5Gclose(g)108                #expect(flags & UInt32(H5P_CRT_ORDER_TRACKED) != 0, "link creation order of \(group)")109            }110        }111        // The dimension placeholders are dimensions, not data: hidden when the file is read back.112        #expect(try out.children(of: "/ints").map(\.name) == ["i16", "i32", "i64", "i8", "u16", "u32", "u64", "u8"])113    }114115    @Test func classicVariantsFromHDF5() async throws {116        let temp = TempDir()117        let single = Fixture.single(temp.file("single.h5"))118        for (variant, version, description) in [("classic", UInt8(1), "NetCDF classic (CDF-1)"), ("64bit-offset", 2, "NetCDF classic (CDF-2, 64-bit offset)"), ("cdf5", 5, "NetCDF classic (CDF-5, 64-bit data)"), ("classic-auto", 1, "NetCDF classic (CDF-1)")] {119            let r = try await convert(single, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string(variant)])120            #expect([UInt8](try Data(contentsOf: r.file).prefix(4)) == [0x43, 0x44, 0x46, version])121            let out = try SciFile(url: r.file)122            #expect(out.formatDescription == description && r.output.facts["sci.format"] == description)123            #expect(try out.node(at: "/m").dataset?.dimensions.map(\.name) == ["dim_3", "dim_4"])124            #expect(try allValues(out, "/m") == (0..<12).map(String.init))125        }126        // The exact bytes of the CDF-1 file, as the specification lays them out.127        let r = try await convert(single, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic")])128        var expected: [UInt8] = [0x43, 0x44, 0x46, 1, 0, 0, 0, 0, 0, 0, 0, 0x0A, 0, 0, 0, 2]129        expected += [0, 0, 0, 5] + Array("dim_3".utf8) + [0, 0, 0] + [0, 0, 0, 3] + [0, 0, 0, 5] + Array("dim_4".utf8) + [0, 0, 0] + [0, 0, 0, 4]130        expected += [0, 0, 0, 0, 0, 0, 0, 0]                               // no global attributes131        expected += [0, 0, 0, 0x0B, 0, 0, 0, 1, 0, 0, 0, 1] + Array("m".utf8) + [0, 0, 0]132        expected += [0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1]                   // two dimensions: 0, 1133        expected += [0, 0, 0, 0, 0, 0, 0, 0]                               // no attributes134        expected += [0, 0, 0, 4, 0, 0, 0, 48, 0, 0, 0, 104]                // NC_INT, vsize 48, begin 104135        for v in 0..<12 { expected += [0, 0, 0, UInt8(v)] }136        #expect([UInt8](try Data(contentsOf: r.file)) == expected)137    }138139    @Test func classicRefusals() async throws {140        let temp = TempDir()141        let everything = Fixture.everything(in: temp.url)142        let grouped = await failure(everything, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic-auto")])143        #expect(grouped.hasPrefix("A setting is not valid. NetCDF classic has no groups, and 26 dataset(s) live inside groups (/array/vectors, /chunked/grid, /compound/particles, /dims/x, /dims/y…). Choose NetCDF-4 as the destination format."), "\(grouped)")144        // Root-level datasets of types classic cannot hold.145        let url = temp.file("types.h5")146        H5.sync {147            let f = C.create(url)148            let vs = C.variableString()149            let words: [UnsafeMutablePointer<CChar>?] = [strdup("a"), strdup("b")]150            words.withUnsafeBytes { _ = C.dataset(f, "words", file: vs, memory: vs, dims: [2], data: $0.baseAddress) }151            words.forEach { free($0) }152            H5Tclose(vs)153            C.numbers(f, "big", ORIX_T_UINT64, [UInt64.max])154            C.numbers(f, "ok", ORIX_T_INT16, [Int16(1), 2])155            H5Fclose(f)156        }157        let strings = await failure(url, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic-auto")])158        #expect(strings == "A setting is not valid. NetCDF classic only holds numbers and characters. 1 variable(s) cannot be stored: /words (string). Choose NetCDF-4 as the destination format.")159        // Unsigned 64-bit integers need CDF-5: an explicit older variant is refused, the automatic choice picks it.160        let numbers = temp.file("numbers.h5")161        H5.sync {162            let f = C.create(numbers)163            C.numbers(f, "big", ORIX_T_UINT64, [UInt64.max, 7])164            C.attribute(f, "names", ORIX_T_INT8, [Int8(1)])165            H5Fclose(f)166        }167        #expect(await failure(numbers, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic")]).contains("The content needs the CDF-5 variant of NetCDF classic"))168        #expect(await failure(numbers, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("64bit-offset")]).contains("The content needs the CDF-5 variant"))169        let auto = try await convert(numbers, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic-auto")])170        #expect(auto.output.facts["sci.format"] == "NetCDF classic (CDF-5, 64-bit data)")171        #expect(try allValues(SciFile(url: auto.file), "/big") == ["18446744073709551615", "7"])172        #expect(await failure(numbers, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("zarr")]).contains("is not a NetCDF variant"))173    }174}175176@Suite struct NetCDFConversionTests {177    @Test(arguments: [UInt8(1), 2, 5]) func classicWriterReproducesIndependentBytes(_ version: UInt8) async throws {178        // The fixture is assembled byte by byte in the test; rewriting it in the same variant must give the same file.179        let temp = TempDir()180        let source = Fixture.classic(temp.file("c.nc"), version: version)181        let variant = version == 1 ? "classic" : version == 2 ? "64bit-offset" : "cdf5"182        let r = try await convert(source, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, options: ["netcdf.format": .string(variant)])183        #expect(try Data(contentsOf: r.file) == Data(contentsOf: source))184        #expect(r.warnings.all.filter { $0.severity == .dataLoss }.isEmpty)185    }186187    @Test func classicToNetCDF4AndBack() async throws {188        let temp = TempDir()189        let sourceURL = Fixture.classic(temp.file("c.nc"), version: 5)190        let source = try SciFile(url: sourceURL)191        // "Automatic" means NetCDF-4 for a classic source.192        let nc4 = try await convert(sourceURL, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, options: ["compression": .string("deflate"), "compression.level": .int(7)], name: "as4")193        let out = try SciFile(url: nc4.file)194        #expect(out.formatDescription == "NetCDF-4 (HDF5)" && nc4.output.facts["sci.dimensions"] == "4" && nc4.output.facts["sci.datasets"] == "9")195        let paths = try datasetPaths(out)196        #expect(paths == ["/lat", "/lon", "/station", "/time", "/temp", "/scalarv", "/pres", "/big", "/flags"])197        // Characters are a one-character string type in NetCDF-4; everything else keeps its exact type.198        try expectSame(source, out, paths: paths.filter { $0 != "/station" })199        #expect(try out.node(at: "/station").dataset?.type == .string(fixedLength: 1))200        #expect(try allValues(out, "/station") == ["A", "B", "C", "D", "E", "F", "", ""])201        #expect(try out.attributes(of: "/").items.map { "\($0.name)=\($0.text())" } == ["title=Classic", "version=7"])202        let t = try #require(try out.node(at: "/temp").dataset)203        #expect(t.dimensions == [SciDimension(name: "time", length: 3, isUnlimited: true, coordinatePath: "/time"), SciDimension(name: "lat", length: 2, isUnlimited: false, coordinatePath: "/lat"),204                                 SciDimension(name: "lon", length: 3, isUnlimited: false, coordinatePath: "/lon")])205        #expect(t.fillValue == "-999" && t.filters.map(\.name) == ["shuffle", "deflate"] && t.filters[1].parameters.first == 7)206        // "strlen" has no coordinate variable: a dimension placeholder, not a dataset.207        #expect(try out.node(at: "/station").dataset?.dimensions[1] == SciDimension(name: "strlen", length: 4, isUnlimited: false, coordinatePath: nil, definitionPath: "/strlen"))208        // And back: automatic means classic for a NetCDF-4 source, the smallest variant that fits (CDF-5 here).209        let back = try await convert(nc4.file, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, name: "back")210        let again = try SciFile(url: back.file)211        #expect(again.formatDescription == "NetCDF classic (CDF-5, 64-bit data)")212        try expectSame(source, again, paths: paths)213        #expect(back.warnings.first("nc-storage-dropped") != nil)214        // To HDF5: the same content, still carrying the NetCDF-4 dimension scales.215        let h5 = try await convert(sourceURL, .netcdf, to: .hdf5, in: temp)216        #expect(h5.file.pathExtension == "h5" && FileTypeDetector.detect(h5.file).format == .hdf5)217        try expectSame(source, try SciFile(url: h5.file), paths: paths.filter { $0 != "/station" })218    }219220    @Test func netCDF4ToHDF5AndRecompression() async throws {221        let temp = TempDir()222        let sourceURL = Fixture.netCDF4(temp.file("raw.nc"))223        let source = try SciFile(url: sourceURL)224        let paths = ["/x", "/temp", "/label", "/model/bias"]225        let h5 = try await convert(sourceURL, .netcdf, to: .hdf5, in: temp)226        let out = try SciFile(url: h5.file)227        try expectSame(source, out, paths: paths)228        let names = try out.children().map(\.name)229        #expect(out.isNetCDF && names == ["x", "temp", "label", "model"])230        let sourceDimensions = try source.node(at: "/temp").dataset?.dimensions231        #expect(try out.node(at: "/temp").dataset?.dimensions == sourceDimensions)232        #expect(try out.node(at: "/temp").dataset?.filters.map(\.name) == ["deflate"] && out.node(at: "/temp").dataset?.chunkShape == [1, 3])233        #expect(try out.node(at: "/model/bias").dataset?.dimensions.first?.coordinatePath == "/x")234        let none = try await convert(sourceURL, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, options: ["netcdf.format": .string("netcdf4"), "compression": .string("none")])235        #expect(try SciFile(url: none.file).node(at: "/temp").dataset?.filters.isEmpty == true)236        // An unlimited dimension keeps its chunking even without compression.237        #expect(try SciFile(url: none.file).node(at: "/temp").dataset?.chunkShape == [1, 3])238        let deflate = try await convert(sourceURL, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation, options: ["netcdf.format": .string("netcdf4"), "compression": .string("deflate"), "compression.level": .int(9), "compression.shuffle": .bool(false)])239        let d = try SciFile(url: deflate.file)240        #expect(try d.node(at: "/x").dataset?.filters.map(\.name) == ["deflate"] && d.node(at: "/x").dataset?.filters.first?.parameters.first == 9)241        try expectSame(source, d, paths: paths)242        // NetCDF-4 with a group cannot become classic.243        #expect(await failure(sourceURL, .netcdf, to: .netcdf, in: temp, operation: ScientificEngine.reformatOperation).contains("NetCDF classic has no groups, and 1 dataset(s) live inside groups (/model/bias)"))244    }245246    @Test func severalUnlimitedDimensionsToClassic() async throws {247        let temp = TempDir()248        let url = temp.file("two.h5")249        H5.sync {250            let f = C.create(url)251            let dcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))252            H5Pset_chunk(dcpl, 2, [hsize_t(2), 2])253            C.numbers(f, "a", ORIX_T_INT32, [Int32(1), 2, 3, 4, 5, 6], dims: [2, 3], maxDims: [H5.unlimited, H5.unlimited], dcpl: dcpl)254            H5Pset_chunk(dcpl, 1, [hsize_t(2)])255            C.numbers(f, "b", ORIX_T_FLOAT, [Float(1.5), 2.5], dims: [2], maxDims: [H5.unlimited], dcpl: dcpl)256            H5Pclose(dcpl)257            C.attribute(f, "weird name/with slash", text: "x")258            let vs = C.variableString()259            let two: [UnsafeMutablePointer<CChar>?] = [strdup("one"), strdup("two")]260            two.withUnsafeBytes { C.attribute(f, "lines", file: vs, memory: vs, dims: [2], data: $0.baseAddress!) }261            two.forEach { free($0) }262            H5Tclose(vs)263            H5Fclose(f)264        }265        let r = try await convert(url, .hdf5, to: .netcdf, in: temp, options: ["netcdf.format": .string("classic")])266        let out = try SciFile(url: r.file)267        // "a" and "b" share the generated dimension of length 2, which leads both: it is the record dimension.268        #expect(try out.node(at: "/a").dataset?.dimensions.map { "\($0.name ?? "")=\($0.length)\($0.isUnlimited ? "u" : "")" } == ["dim_2=2u", "dim_3=3"])269        #expect(try out.node(at: "/b").dataset?.dimensions.map { "\($0.name ?? "")=\($0.length)\($0.isUnlimited ? "u" : "")" } == ["dim_2=2u"])270        #expect(r.warnings.first("nc-unlimited-fixed")?.message == "The unlimited dimension dim_3 is written with its current fixed length 3: NetCDF classic allows one unlimited dimension, used first.")271        #expect(try allValues(out, "/a") == ["1", "2", "3", "4", "5", "6"] && allValues(out, "/b") == ["1.5", "2.5"])272        #expect(r.warnings.first("nc-string-attributes-joined")?.message.hasPrefix("1 attribute(s)") == true)273        #expect(r.warnings.first("nc-storage-dropped") != nil)274        #expect(try out.attributes(of: "/").items.map { "\($0.name)=\($0.text())" } == ["lines=one\ntwo", "weird name_with slash=x"])275    }276}277278@Suite struct RepackTests {279    @Test func repackKeepsEverythingItCan() async throws {280        let temp = TempDir()281        let sourceURL = Fixture.everything(in: temp.url)282        let source = try SciFile(url: sourceURL)283        let r = try await convert(sourceURL, .hdf5, to: .hdf5, in: temp, operation: ScientificEngine.repackOperation, options: ["compression": .string("deflate"), "compression.level": .int(5)])284        let out = try SciFile(url: r.file)285        #expect(out.formatDescription == "HDF5" && !out.isNetCDF)286        let paths = try datasetPaths(out)287        // 28 names in the source: the reference dataset is left out; the hard link and the soft link are links again.288        #expect(paths.count == 27 && !paths.contains("/refs/objref") && r.output.facts["sci.datasets"] == "25" && r.output.facts["sci.links"] == "2")289        try expectSame(source, out, paths: paths)290        #expect(r.warnings.all("sci-dataset-skipped").map(\.message) == ["The dataset /refs/objref is not in the output: it holds object or region references."])291        #expect(r.warnings.first("sci-external-link") != nil && r.warnings.first("sci-dangling-link") != nil)292        #expect(try out.children(of: "/links").map { "\($0.name):\($0.softLinkTarget ?? "-")" } == ["alias:-", "soft:/scalar"])293        #expect(out.backend.identity(of: "/links/alias") == out.backend.identity(of: "/ints/i32"))294        // Types that only HDF5 has survive: compound with text, array, enumeration, half floats.295        #expect(try out.node(at: "/compound/particles").dataset?.type.description == "compound {id: int32, mass: float64, name: string(6), flags: uint8}")296        #expect(try allValues(out, "/compound/particles") == ["1|1.5|p1|1", "2|2.25|proton|200", "-3|0.0||0"])297        #expect(try allValues(out, "/floats/f16") == ["0.5", "-2.0"] && allValues(out, "/enum/colors") == ["RED", "BLUE", "GREEN", "BLUE"])298        // Dimension scales are attached again; storage settings follow the options.299        let grid = try #require(try out.node(at: "/chunked/grid").dataset)300        #expect(grid.dimensions == [SciDimension(name: "y", length: 20, isUnlimited: false, coordinatePath: "/dims/y"), SciDimension(name: "x", length: 30, isUnlimited: false, coordinatePath: "/dims/x")])301        #expect(grid.chunkShape == [5, 10] && grid.filters.map(\.name) == ["shuffle", "deflate"] && grid.filters[1].parameters.first == 5 && grid.fillValue == "-999")302        #expect(try out.node(at: "/ints/i64").dataset?.filters.map(\.name) == ["shuffle", "deflate"])303        #expect(try out.node(at: "/ints/i8").dataset?.filters.map(\.name) == ["deflate"], "one-byte elements are not shuffled")304        #expect(try out.node(at: "/scalar").dataset?.filters.isEmpty == true, "a scalar cannot be chunked")305        #expect(try out.node(at: "/unlimited/series").dataset?.dimensions.first?.isUnlimited == true)306        #expect(try out.node(at: "/floats/f64be").dataset?.type == .float(bits: 64))307        // "Same as the source" and "none".308        let same = try SciFile(url: try await convert(sourceURL, .hdf5, to: .hdf5, in: temp, operation: ScientificEngine.repackOperation).file)309        #expect(try same.node(at: "/chunked/grid").dataset?.filters[1].parameters.first == 6 && same.node(at: "/ints/i64").dataset?.layout == .contiguous)310        let none = try SciFile(url: try await convert(sourceURL, .hdf5, to: .hdf5, in: temp, operation: ScientificEngine.repackOperation, options: ["compression": .string("none")]).file)311        #expect(try none.node(at: "/chunked/grid").dataset?.filters.isEmpty == true && none.node(at: "/unlimited/series").dataset?.chunkShape == [4])312        try expectSame(source, none, paths: paths)313    }314315    @Test func storageThatIsNeverFollowed() async throws {316        let temp = TempDir()317        let url = temp.file("storage.h5")318        try Data([1, 0, 0, 0, 2, 0, 0, 0]).write(to: temp.file("raw.bin"))319        H5.sync {320            let f = C.create(url)321            // Raw data in an external file.322            let dcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))323            H5Pset_external(dcpl, "raw.bin", 0, 8)324            C.dataset(f, "external", file: H5.type(ORIX_T_INT32), memory: H5.type(ORIX_T_INT32), dims: [2], dcpl: dcpl, data: nil)325            H5Pclose(dcpl)326            // A virtual dataset mapped onto another file.327            let vcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))328            var n = hsize_t(2)329            let vspace = H5Screate_simple(1, &n, nil), sspace = H5Screate_simple(1, &n, nil)330            H5Pset_virtual(vcpl, vspace, "other.h5", "/data", sspace)331            C.dataset(f, "virtual", file: H5.type(ORIX_T_INT32), memory: H5.type(ORIX_T_INT32), dims: [2], dcpl: vcpl, data: nil)332            H5Sclose(vspace); H5Sclose(sspace); H5Pclose(vcpl)333            C.numbers(f, "plain", ORIX_T_INT32, [Int32(5), 6])334            H5Fclose(f)335            let other = C.create(temp.file("other.h5"))336            C.numbers(other, "data", ORIX_T_INT32, [Int32(111), 222])337            H5Fclose(other)338        }339        let file = try SciFile(url: url)340        let external = try #require(try file.node(at: "/external").dataset), virtual = try #require(try file.node(at: "/virtual").dataset)341        #expect(external.layout == .external && external.unreadableReason == "its raw data is stored in external files (never followed)")342        #expect(virtual.layout == .virtual && virtual.unreadableReason == "it is a virtual dataset, whose data may live in other files (never followed)")343        #expect(throws: ConversionError.malformedInput("/external cannot be read: its raw data is stored in external files (never followed).")) { _ = try file.backend.read("/external", SciSelection(all: [2])) }344        #expect(throws: ConversionError.self) { _ = try file.backend.read("/virtual", SciSelection(all: [2])) }345        #expect(try file.preview("/external").note == "The values cannot be read: its raw data is stored in external files (never followed).")346        #expect(await failure(url, .hdf5, to: .csv, in: temp, options: ["variables": .string("/virtual")]).contains("/virtual cannot be read: it is a virtual dataset"))347        let r = try await convert(url, .hdf5, to: .netcdf, in: temp)348        #expect(r.warnings.all("sci-dataset-skipped").map(\.message).sorted() == [349            "The dataset /external is not in the output: its raw data is stored in external files (never followed).",350            "The dataset /virtual is not in the output: it is a virtual dataset, whose data may live in other files (never followed).",351        ])352        #expect(try datasetPaths(SciFile(url: r.file)) == ["/plain"])353        let i = try await ScientificEngine().inspect(url, format: .hdf5)354        #expect(i.fields.first { $0.key == "/external" }?.value.contains("unreadable: its raw data is stored in external files") == true)355    }356357    @Test func cancellationAndProgress() async throws {358        let temp = TempDir()359        let url = temp.file("large.h5")360        H5.sync {361            let f = C.create(url)362            C.numbers(f, "v", ORIX_T_DOUBLE, (0..<4_000_000).map { Double($0) })363            H5Fclose(f)364        }365        let r = try await convert(url, .hdf5, to: .hdf5, in: temp, operation: ScientificEngine.repackOperation, options: ["compression": .string("deflate"), "compression.level": .int(1)])366        #expect(r.progress.fractions.count >= 3 && r.progress.fractions == r.progress.fractions.sorted() && r.progress.fractions.last == 1)367        #expect(r.output.facts["sci.elements"] == "4000000")368        for destination in [FormatID.hdf5, .csv, .parquet] {369            let work = temp.sub("cancel-\(destination.rawValue)")370            let task = Task {371                try await ScientificEngine().convert(ConversionRequest(source: url, sourceFormat: .hdf5, destinationFormat: destination, operation: destination == .hdf5 ? ScientificEngine.repackOperation : nil),372                                                     context: ConversionContext(workDirectory: work))373            }374            task.cancel()375            await #expect(throws: CancellationError.self) { _ = try await task.value }376        }377    }378}379
Files main
Menu

Theme