Repository Public License
1// ORIX Convert — Packages/EngineScientific/Tests/EngineScientificTests/InspectTests.swift2// Description: Dataset inspection and reading against fixtures with known contents: the tree, shapes, dimensions,3// types, chunking, filters, fill values, attributes, dimension scales, links, hyperslab reads.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import Testing7import CHDF58import CHDF5Shim9import ConvertCore10@testable import EngineScientific1112func values(_ file: SciFile, _ path: String, start: [UInt64]? = nil, count: [UInt64]? = nil, stride: [UInt64]? = nil) throws -> [String] {13 try file.preview(path, start: start, count: count, stride: stride, maxElements: 100_000).values14}1516@Suite struct HDF5InspectTests {17 @Test func libraryIsHardened() {18 H5.sync {19 #expect(orix_h5_plugins_disabled() == 1)20 #expect(H5.version == "2.2.0")21 }22 #expect(ProcessInfo.processInfo.environment["HDF5_PLUGIN_PRELOAD"] == nil || getenv("HDF5_PLUGIN_PRELOAD") != nil)23 #expect(String(cString: getenv("HDF5_PLUGIN_PRELOAD")) == "::")24 }2526 @Test func tree() throws {27 let temp = TempDir()28 let file = try SciFile(url: Fixture.everything(in: temp.url))29 #expect(file.formatDescription == "HDF5" && !file.isNetCDF)30 let root = try file.children()31 #expect(root.map(\.name) == ["array", "chunked", "compound", "dims", "empty", "enum", "floats", "ints", "links", "refs", "scalar", "strings", "time", "unlimited"])32 #expect(root.first { $0.name == "ints" }?.childCount == 8)33 #expect(root.first { $0.name == "ints" }?.attributeCount == 1)34 #expect(try file.node(at: "/").attributeCount == 4)35 #expect(try file.children(of: "/ints", offset: 2, limit: 3).map(\.name) == ["i64", "i8", "u16"])36 let (all, complete) = try file.tree()37 #expect(complete && all.filter { $0.kind == .dataset }.count == 28 && all.filter { $0.kind == .group }.count == 13)38 #expect(try file.tree(maxNodes: 5).isComplete == false)39 #expect(throws: ConversionError.invalidOption("The file has no object at /nope.")) { _ = try file.node(at: "/nope") }40 }4142 @Test func datasetDescriptions() throws {43 let temp = TempDir()44 let file = try SciFile(url: Fixture.everything(in: temp.url))45 func info(_ path: String) throws -> SciDatasetInfo { try #require(try file.node(at: path).dataset) }46 // Every integer width and signedness, floats, strings.47 for (name, bits, signed) in [("i8", 8, true), ("u8", 8, false), ("i16", 16, true), ("u16", 16, false), ("i32", 32, true), ("u32", 32, false), ("i64", 64, true), ("u64", 64, false)] {48 #expect(try info("/ints/\(name)").type == .integer(bits: bits, signed: signed))49 #expect(try info("/ints/\(name)").shape == [4])50 }51 #expect(try info("/floats/f32").type == .float(bits: 32) && info("/floats/f64").type == .float(bits: 64) && info("/floats/f16").type == .float(bits: 16))52 #expect(try info("/floats/f32").shape == [2, 3] && info("/floats/f32").layout == .contiguous)53 #expect(try info("/strings/fixed").type == .string(fixedLength: 8) && info("/strings/variable").type == .string(fixedLength: nil))54 #expect(try info("/compound/particles").type.description == "compound {id: int32, mass: float64, name: string(6), flags: uint8}")55 #expect(try info("/enum/colors").type == .enumeration(bits: 8, signed: true, members: ["RED": 0, "GREEN": 1, "BLUE": 2]))56 #expect(try info("/array/vectors").type == .array(dimensions: [3], base: .integer(bits: 32, signed: true)))57 #expect(try info("/refs/objref").type == .reference)58 #expect(try info("/scalar").shape == [] && info("/scalar").elementCount == 1)59 // Chunking, filters, fill value, dimension scales.60 let grid = try info("/chunked/grid")61 #expect(grid.layout == .chunked && grid.chunkShape == [5, 10])62 #expect(grid.filters.map(\.name) == ["shuffle", "deflate"] && grid.filters[1].parameters.first == 6 && grid.filters.allSatisfy(\.isAvailable))63 #expect(grid.fillValue == "-999")64 #expect(grid.dimensions == [SciDimension(name: "y", length: 20, isUnlimited: false, coordinatePath: "/dims/y"), SciDimension(name: "x", length: 30, isUnlimited: false, coordinatePath: "/dims/x")])65 #expect((grid.storedBytes ?? 0) > 0 && (grid.storedBytes ?? 0) < 1200, "deflate + shuffle shrink 1200 bytes")66 #expect(try info("/dims/x").isCoordinate && info("/dims/x").dimensions.first?.name == "x")67 #expect(try !info("/ints/i8").isCoordinate && info("/ints/i8").dimensions.first?.name == nil)68 let series = try info("/unlimited/series")69 #expect(series.dimensions == [SciDimension(name: nil, length: 7, isUnlimited: true)] && series.chunkShape == [4])70 // CF hints, as read.71 let c = try file.conventions(of: "/chunked/grid")72 #expect(c.units == "K" && c.standardName == "air_temperature" && c.scaleFactor == 0.01 && c.addOffset == 273.15 && c.fillValue == "-999" && c.isPacked)73 #expect(try file.conventions(of: "/time/t").calendar == "standard")74 }7576 @Test func attributes() throws {77 let temp = TempDir()78 let file = try SciFile(url: Fixture.everything(in: temp.url))79 let (items, total) = try file.attributes(of: "/")80 #expect(total == 4)81 let byName = Dictionary(uniqueKeysWithValues: items.map { ($0.name, $0) })82 #expect(byName["title"]?.text() == "Everything" && byName["title"]?.type == .string(fixedLength: 10) && byName["title"]?.shape == [])83 #expect(byName["version"]?.text() == "3" && byName["version"]?.type == .integer(bits: 32, signed: true))84 #expect(byName["ratios"]?.text() == "0.5, 1.5, 2.5" && byName["ratios"]?.shape == [3])85 #expect(byName["tags"]?.text() == "\"alpha\", \"βeta\"" && byName["tags"]?.type == .string(fixedLength: nil))86 // Paging and bounded values.87 #expect(try file.attributes(of: "/", offset: 1, limit: 2).items.count == 2)88 let bounded = try file.attributes(of: "/", maxElements: 2).items.first { $0.name == "ratios" }89 #expect(bounded?.isTruncated == true && bounded?.text() == "0.5, 1.5, … (3 values)")90 let page = try #require(try file.attribute("ratios", of: "/", offset: 1, maxElements: 5))91 #expect(page.values.count == 2 && page.values.text(0) == "1.5" && page.values.text(1) == "2.5" && page.totalElements == 3)92 // Dimension-scale bookkeeping is structure, not user metadata.93 #expect(try file.attributes(of: "/chunked/grid").items.map(\.name) == ["_FillValue", "add_offset", "scale_factor", "standard_name", "units"])94 #expect(try file.attributes(of: "/dims/x").items.map(\.name) == ["units"])95 #expect(try file.node(at: "/dims/x").attributeCount == 1)96 }9798 @Test func valuesOfEveryType() throws {99 let temp = TempDir()100 let file = try SciFile(url: Fixture.everything(in: temp.url))101 #expect(try values(file, "/ints/i8") == ["-128", "-1", "0", "127"])102 #expect(try values(file, "/ints/u8") == ["0", "1", "128", "255"])103 #expect(try values(file, "/ints/i16") == ["-32768", "-1", "0", "32767"])104 #expect(try values(file, "/ints/u16") == ["0", "1", "32768", "65535"])105 #expect(try values(file, "/ints/i32") == ["-2147483648", "-1", "0", "2147483647"])106 #expect(try values(file, "/ints/u32") == ["0", "1", "2147483648", "4294967295"])107 #expect(try values(file, "/ints/i64") == ["-9223372036854775808", "-1", "0", "9223372036854775807"])108 #expect(try values(file, "/ints/u64") == ["0", "1", "9223372036854775808", "18446744073709551615"])109 #expect(try values(file, "/floats/f32") == ["0.1", "-1.5", "3.25", "1e-10", "1e+20", "NaN"])110 #expect(try values(file, "/floats/f64") == ["0.1", "-1.5", "3.25", "1e-300", "1e+300", "Infinity"])111 #expect(try values(file, "/floats/f64be") == ["1.0", "2.0", "3.0"])112 #expect(try values(file, "/floats/f16") == ["0.5", "-2.0"])113 #expect(try values(file, "/strings/fixed") == ["abc", "defghijk", ""])114 #expect(try values(file, "/strings/spacepad") == ["ab", "cde"])115 #expect(try values(file, "/strings/variable") == ["one", "deux é", ""])116 #expect(try values(file, "/enum/colors") == ["RED", "BLUE", "GREEN", "BLUE"])117 #expect(try values(file, "/array/vectors") == ["[1,2,3]", "[4,5,6]"])118 #expect(try values(file, "/scalar") == ["42.5"])119 let particles = try file.preview("/compound/particles")120 #expect(particles.memberNames == ["id", "mass", "name", "flags"])121 #expect(particles.values == ["1\t1.5\tp1\t1", "2\t2.25\tproton\t200", "-3\t0.0\t\t0"])122 let refs = try file.preview("/refs/objref")123 #expect(refs.values.isEmpty && refs.note == "The values cannot be shown: object and region references are not converted.")124 }125126 @Test func hyperslabs() throws {127 let temp = TempDir()128 let file = try SciFile(url: Fixture.everything(in: temp.url))129 // Rows 3, 8, 13 and columns 28, 29 of the chunked grid (crossing chunk boundaries).130 let p = try file.preview("/chunked/grid", start: [3, 28], count: [3, 2], stride: [5, 1])131 #expect(p.values == ["328", "329", "828", "829", "1328", "1329"])132 #expect(p.selection == SciSelection(start: [3, 28], count: [3, 2], stride: [5, 1]) && !p.isTruncated)133 // The default window starts at the origin and is cut to the element bound, as a rectangle.134 let head = try file.preview("/chunked/grid", maxElements: 60)135 #expect(head.selection.count == [2, 30] && head.isTruncated && head.values.first == "-999" && head.values.last == "129")136 // A window past the end is clipped; an invalid one is refused.137 #expect(try values(file, "/unlimited/series", start: [5], count: [10]) == ["7.5", "9.0"])138 #expect(try values(file, "/unlimited/series", start: [1], stride: [3]) == ["1.5", "6.0"])139 #expect(try values(file, "/unlimited/series", start: [99]) == [])140 #expect(throws: ConversionError.self) { _ = try file.preview("/chunked/grid", start: [0]) }141 #expect(throws: ConversionError.self) { _ = try file.backend.read("/chunked/grid", SciSelection(start: [19, 0], count: [2, 1], stride: [1, 1])) }142 #expect(throws: ConversionError.self) { _ = try file.backend.read("/chunked/grid", SciSelection(start: [0, 0], count: [1, 1], stride: [0, 1])) }143 #expect(throws: ConversionError.self) { _ = try file.preview("/ints") }144 }145146 @Test func linksAreReportedAndNeverFollowed() throws {147 let temp = TempDir()148 let file = try SciFile(url: Fixture.everything(in: temp.url))149 // Two independent defences: the library's external-link class is replaced by one that never traverses, and150 // every access property list the engine uses carries a callback that refuses the traversal.151 #expect(H5.sync { orix_h5_external_links_disabled() } == 1)152 let before = H5.sync { orix_h5_external_refusals() }153 let links = try file.children(of: "/links")154 let byName = Dictionary(uniqueKeysWithValues: links.map { ($0.name, $0.kind) })155 #expect(byName["ext"] == .externalLink(file: "other.h5", path: "/data"))156 #expect(byName["dangling"] == .danglingLink(target: "/nowhere"))157 #expect(byName["soft"] == .dataset && byName["alias"] == .dataset)158 #expect(links.first { $0.name == "soft" }?.softLinkTarget == "/scalar" && links.first { $0.name == "alias" }?.softLinkTarget == nil)159 #expect(file.notes.contains { $0.contains("/links/ext is a link to another file (other.h5)") })160 // The target file exists and holds [111, 222]; nothing reaches it, by any route.161 #expect(try SciFile(url: temp.file("other.h5")).preview("/data").values == ["111", "222"])162 #expect(throws: ConversionError.self) { _ = try file.backend.read("/links/ext", SciSelection(all: [2])) }163 #expect(throws: ConversionError.self) { _ = try file.node(at: "/links/ext/anything") }164 #expect(throws: ConversionError.self) { _ = try file.attributes(of: "/links/ext") }165 #expect(throws: ConversionError.self) { _ = try file.children(of: "/links/ext") }166 #expect(throws: ConversionError.self) { _ = try file.preview("/links/ext") }167 // Even a raw open with default property lists (which the engine never uses on a source) is refused.168 let h5 = try #require(file.backend as? H5Backend)169 H5.sync {170 #expect(H5Dopen2(h5.file, "/links/ext", 0) < 0)171 #expect(H5Oopen(h5.file, "/links/ext", 0) < 0)172 #expect(H5Dopen2(h5.file, "/links/ext", h5.dapl) < 0)173 _ = H5.lastError()174 #expect(orix_h5_external_refusals() >= before + 3)175 }176 // The soft link inside the file works; the hard-link alias is the same object.177 #expect(try values(file, "/links/soft") == ["42.5"])178 #expect(file.backend.identity(of: "/links/alias") == file.backend.identity(of: "/ints/i32"))179 #expect(file.backend.identity(of: "/links/alias") != file.backend.identity(of: "/ints/i64"))180 }181182 @Test func engineInspection() async throws {183 let temp = TempDir()184 let i = try await ScientificEngine().inspect(Fixture.everything(in: temp.url), format: .hdf5)185 func field(_ group: String, _ key: String) -> String? { i.fields.first { $0.group == group && $0.key == key }?.value }186 #expect(field("Scientific data", "Container") == "HDF5")187 #expect(field("Scientific data", "Groups") == "13" && field("Scientific data", "Datasets") == "28")188 #expect(field("Scientific data", "Links not followed") == "/links/dangling, /links/ext")189 #expect(field("Scientific data", "Dimensions") == "y = 20, x = 30, t = 5")190 #expect(field("Global attributes", "title") == "Everything" && field("Global attributes", "ratios") == "0.5, 1.5, 2.5")191 #expect(field("Datasets", "/chunked/grid") == "int16; shape y=20 × x=30; chunks 5 × 10; filters shuffle + deflate(6); fill -999; units “K”; standard_name air_temperature; packed (scale_factor 0.01, add_offset 273.15); 5 attribute(s)")192 #expect(field("Datasets", "/unlimited/series") == "float32; shape 7 (unlimited); chunks 4")193 #expect(field("Datasets", "/time/t") == "float64; shape t=5; coordinate variable; units “hours since 2000-01-01 00:00:00”; calendar standard; 2 attribute(s)")194 #expect(field("Datasets", "/scalar") == "float64; scalar; units “m”; 1 attribute(s)")195 #expect(i.entryCount == 28 && i.hasMetadata == true)196 #expect(i.notes.contains { $0.contains("external links are never followed") })197 }198}199200@Suite struct NetCDFInspectTests {201 @Test(arguments: [UInt8(1), 2, 5]) func classicFromIndependentBytes(_ version: UInt8) throws {202 let temp = TempDir()203 let file = try SciFile(url: Fixture.classic(temp.file("c\(version).nc"), version: version))204 #expect(file.formatDescription == ["", "NetCDF classic (CDF-1)", "NetCDF classic (CDF-2, 64-bit offset)", "", "", "NetCDF classic (CDF-5, 64-bit data)"][Int(version)])205 #expect(file.isNetCDF)206 let nodes = try file.children()207 #expect(nodes.map(\.name) == ["lat", "lon", "station", "time", "temp", "scalarv", "pres"] + (version == 5 ? ["big", "flags"] : []))208 let temp_ = try #require(nodes[4].dataset)209 #expect(temp_.type == .integer(bits: 16, signed: true))210 #expect(temp_.dimensions == [SciDimension(name: "time", length: 3, isUnlimited: true, coordinatePath: "/time", definitionPath: "/time"),211 SciDimension(name: "lat", length: 2, isUnlimited: false, coordinatePath: "/lat", definitionPath: "/lat"),212 SciDimension(name: "lon", length: 3, isUnlimited: false, coordinatePath: "/lon", definitionPath: "/lon")])213 #expect(temp_.fillValue == "-999")214 #expect(nodes[0].dataset?.isCoordinate == true && nodes[4].dataset?.isCoordinate == false)215 #expect(nodes[2].dataset?.type == .char && nodes[2].dataset?.shape == [2, 4])216 #expect(nodes[5].dataset?.shape == [])217 // Attributes.218 let global = try file.attributes(of: "/").items219 #expect(global.map(\.name) == ["title", "version"] && global[0].text() == "Classic" && global[1].text() == "7" && global[1].type == .integer(bits: 32, signed: true))220 let ta = try file.attributes(of: "/temp").items221 #expect(ta.map { "\($0.name)=\($0.text())" } == ["_FillValue=-999", "scale_factor=0.5", "units=K"])222 #expect(try file.conventions(of: "/time").units == "hours since 2020-01-01 00:00:00")223 // Values: fixed variables, record variables, hyperslabs across records.224 #expect(try values(file, "/lat") == ["45.0", "46.5"])225 #expect(try values(file, "/lon") == ["-75.0", "-74.5", "-74.0"])226 #expect(try values(file, "/scalarv") == ["3.5"])227 #expect(try values(file, "/time") == ["0", "6", "12"])228 #expect(try values(file, "/pres") == ["1000.5", "999.25", "998.0"])229 #expect(try values(file, "/station") == ["A", "B", "C", "D", "E", "F", "\0", "\0"])230 var expected: [String] = []231 for t in 0..<3 { for la in 0..<2 { for lo in 0..<3 { expected.append(String(Fixture.classicTemp(t, la, lo))) } } }232 #expect(try values(file, "/temp") == expected)233 #expect(try values(file, "/temp", start: [0, 1, 0], count: [3, 1, 2], stride: [2, 1, 2]) == ["10", "12", "210", "212"])234 #expect(try values(file, "/temp", start: [2, 0, 1], count: [1, 2, 1]) == ["201", "211"])235 #expect(try values(file, "/pres", start: [2]) == ["998.0"])236 if version == 5 {237 #expect(try values(file, "/big") == ["-9000000000", "9000000000"])238 #expect(try values(file, "/flags") == ["1", "200", "255"])239 #expect(nodes[7].dataset?.type == .integer(bits: 64, signed: true) && nodes[8].dataset?.type == .integer(bits: 8, signed: false))240 #expect(try file.attributes(of: "/big").items.first?.text() == "18446744073709551615")241 }242 }243244 @Test func netCDF4BuiltWithRawCalls() async throws {245 let temp = TempDir()246 let file = try SciFile(url: Fixture.netCDF4(temp.file("raw.nc")))247 #expect(file.formatDescription == "NetCDF-4 (HDF5)" && file.isNetCDF)248 // The dimension-only dataset "time" is a dimension, not a variable; creation order is kept.249 #expect(try file.children().map(\.name) == ["x", "temp", "label", "model"])250 let t = try #require(try file.node(at: "/temp").dataset)251 #expect(t.dimensions == [SciDimension(name: "time", length: 2, isUnlimited: true, coordinatePath: nil, definitionPath: "/time"),252 SciDimension(name: "x", length: 3, isUnlimited: false, coordinatePath: "/x", definitionPath: "/x")])253 #expect(t.filters.map(\.name) == ["deflate"] && t.chunkShape == [1, 3])254 #expect(try file.node(at: "/model/bias").dataset?.dimensions.first?.name == "x")255 // NetCDF-4 internals are not user attributes.256 #expect(try file.attributes(of: "/").items.map(\.name) == ["Conventions"])257 #expect(try file.attributes(of: "/x").items.map(\.name) == ["units"])258 #expect(try values(file, "/temp") == ["10.0", "11.0", "12.0", "20.0", "21.0", "22.0"])259 let i = try await ScientificEngine().inspect(file.url, format: .netcdf)260 #expect(i.fields.first { $0.key == "Dimensions" }?.value == "x = 3, time = 2 (unlimited)")261 #expect(i.fields.first { $0.key == "Container" }?.value == "NetCDF-4 (HDF5)")262 }263}264