SPB Git

Repository Public License

InspectTests.swift 19.3 KB 264 lines swift

1// ORIX Convert — Packages/EngineScientific/Tests/EngineScientificTests/InspectTests.swift2// Description: Dataset inspection and reading against fixtures with known contents: the tree, shapes, dimensions,3//              types, chunking, filters, fill values, attributes, dimension scales, links, hyperslab reads.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import Testing7import CHDF58import CHDF5Shim9import ConvertCore10@testable import EngineScientific1112func values(_ file: SciFile, _ path: String, start: [UInt64]? = nil, count: [UInt64]? = nil, stride: [UInt64]? = nil) throws -> [String] {13    try file.preview(path, start: start, count: count, stride: stride, maxElements: 100_000).values14}1516@Suite struct HDF5InspectTests {17    @Test func libraryIsHardened() {18        H5.sync {19            #expect(orix_h5_plugins_disabled() == 1)20            #expect(H5.version == "2.2.0")21        }22        #expect(ProcessInfo.processInfo.environment["HDF5_PLUGIN_PRELOAD"] == nil || getenv("HDF5_PLUGIN_PRELOAD") != nil)23        #expect(String(cString: getenv("HDF5_PLUGIN_PRELOAD")) == "::")24    }2526    @Test func tree() throws {27        let temp = TempDir()28        let file = try SciFile(url: Fixture.everything(in: temp.url))29        #expect(file.formatDescription == "HDF5" && !file.isNetCDF)30        let root = try file.children()31        #expect(root.map(\.name) == ["array", "chunked", "compound", "dims", "empty", "enum", "floats", "ints", "links", "refs", "scalar", "strings", "time", "unlimited"])32        #expect(root.first { $0.name == "ints" }?.childCount == 8)33        #expect(root.first { $0.name == "ints" }?.attributeCount == 1)34        #expect(try file.node(at: "/").attributeCount == 4)35        #expect(try file.children(of: "/ints", offset: 2, limit: 3).map(\.name) == ["i64", "i8", "u16"])36        let (all, complete) = try file.tree()37        #expect(complete && all.filter { $0.kind == .dataset }.count == 28 && all.filter { $0.kind == .group }.count == 13)38        #expect(try file.tree(maxNodes: 5).isComplete == false)39        #expect(throws: ConversionError.invalidOption("The file has no object at /nope.")) { _ = try file.node(at: "/nope") }40    }4142    @Test func datasetDescriptions() throws {43        let temp = TempDir()44        let file = try SciFile(url: Fixture.everything(in: temp.url))45        func info(_ path: String) throws -> SciDatasetInfo { try #require(try file.node(at: path).dataset) }46        // Every integer width and signedness, floats, strings.47        for (name, bits, signed) in [("i8", 8, true), ("u8", 8, false), ("i16", 16, true), ("u16", 16, false), ("i32", 32, true), ("u32", 32, false), ("i64", 64, true), ("u64", 64, false)] {48            #expect(try info("/ints/\(name)").type == .integer(bits: bits, signed: signed))49            #expect(try info("/ints/\(name)").shape == [4])50        }51        #expect(try info("/floats/f32").type == .float(bits: 32) && info("/floats/f64").type == .float(bits: 64) && info("/floats/f16").type == .float(bits: 16))52        #expect(try info("/floats/f32").shape == [2, 3] && info("/floats/f32").layout == .contiguous)53        #expect(try info("/strings/fixed").type == .string(fixedLength: 8) && info("/strings/variable").type == .string(fixedLength: nil))54        #expect(try info("/compound/particles").type.description == "compound {id: int32, mass: float64, name: string(6), flags: uint8}")55        #expect(try info("/enum/colors").type == .enumeration(bits: 8, signed: true, members: ["RED": 0, "GREEN": 1, "BLUE": 2]))56        #expect(try info("/array/vectors").type == .array(dimensions: [3], base: .integer(bits: 32, signed: true)))57        #expect(try info("/refs/objref").type == .reference)58        #expect(try info("/scalar").shape == [] && info("/scalar").elementCount == 1)59        // Chunking, filters, fill value, dimension scales.60        let grid = try info("/chunked/grid")61        #expect(grid.layout == .chunked && grid.chunkShape == [5, 10])62        #expect(grid.filters.map(\.name) == ["shuffle", "deflate"] && grid.filters[1].parameters.first == 6 && grid.filters.allSatisfy(\.isAvailable))63        #expect(grid.fillValue == "-999")64        #expect(grid.dimensions == [SciDimension(name: "y", length: 20, isUnlimited: false, coordinatePath: "/dims/y"), SciDimension(name: "x", length: 30, isUnlimited: false, coordinatePath: "/dims/x")])65        #expect((grid.storedBytes ?? 0) > 0 && (grid.storedBytes ?? 0) < 1200, "deflate + shuffle shrink 1200 bytes")66        #expect(try info("/dims/x").isCoordinate && info("/dims/x").dimensions.first?.name == "x")67        #expect(try !info("/ints/i8").isCoordinate && info("/ints/i8").dimensions.first?.name == nil)68        let series = try info("/unlimited/series")69        #expect(series.dimensions == [SciDimension(name: nil, length: 7, isUnlimited: true)] && series.chunkShape == [4])70        // CF hints, as read.71        let c = try file.conventions(of: "/chunked/grid")72        #expect(c.units == "K" && c.standardName == "air_temperature" && c.scaleFactor == 0.01 && c.addOffset == 273.15 && c.fillValue == "-999" && c.isPacked)73        #expect(try file.conventions(of: "/time/t").calendar == "standard")74    }7576    @Test func attributes() throws {77        let temp = TempDir()78        let file = try SciFile(url: Fixture.everything(in: temp.url))79        let (items, total) = try file.attributes(of: "/")80        #expect(total == 4)81        let byName = Dictionary(uniqueKeysWithValues: items.map { ($0.name, $0) })82        #expect(byName["title"]?.text() == "Everything" && byName["title"]?.type == .string(fixedLength: 10) && byName["title"]?.shape == [])83        #expect(byName["version"]?.text() == "3" && byName["version"]?.type == .integer(bits: 32, signed: true))84        #expect(byName["ratios"]?.text() == "0.5, 1.5, 2.5" && byName["ratios"]?.shape == [3])85        #expect(byName["tags"]?.text() == "\"alpha\", \"βeta\"" && byName["tags"]?.type == .string(fixedLength: nil))86        // Paging and bounded values.87        #expect(try file.attributes(of: "/", offset: 1, limit: 2).items.count == 2)88        let bounded = try file.attributes(of: "/", maxElements: 2).items.first { $0.name == "ratios" }89        #expect(bounded?.isTruncated == true && bounded?.text() == "0.5, 1.5, … (3 values)")90        let page = try #require(try file.attribute("ratios", of: "/", offset: 1, maxElements: 5))91        #expect(page.values.count == 2 && page.values.text(0) == "1.5" && page.values.text(1) == "2.5" && page.totalElements == 3)92        // Dimension-scale bookkeeping is structure, not user metadata.93        #expect(try file.attributes(of: "/chunked/grid").items.map(\.name) == ["_FillValue", "add_offset", "scale_factor", "standard_name", "units"])94        #expect(try file.attributes(of: "/dims/x").items.map(\.name) == ["units"])95        #expect(try file.node(at: "/dims/x").attributeCount == 1)96    }9798    @Test func valuesOfEveryType() throws {99        let temp = TempDir()100        let file = try SciFile(url: Fixture.everything(in: temp.url))101        #expect(try values(file, "/ints/i8") == ["-128", "-1", "0", "127"])102        #expect(try values(file, "/ints/u8") == ["0", "1", "128", "255"])103        #expect(try values(file, "/ints/i16") == ["-32768", "-1", "0", "32767"])104        #expect(try values(file, "/ints/u16") == ["0", "1", "32768", "65535"])105        #expect(try values(file, "/ints/i32") == ["-2147483648", "-1", "0", "2147483647"])106        #expect(try values(file, "/ints/u32") == ["0", "1", "2147483648", "4294967295"])107        #expect(try values(file, "/ints/i64") == ["-9223372036854775808", "-1", "0", "9223372036854775807"])108        #expect(try values(file, "/ints/u64") == ["0", "1", "9223372036854775808", "18446744073709551615"])109        #expect(try values(file, "/floats/f32") == ["0.1", "-1.5", "3.25", "1e-10", "1e+20", "NaN"])110        #expect(try values(file, "/floats/f64") == ["0.1", "-1.5", "3.25", "1e-300", "1e+300", "Infinity"])111        #expect(try values(file, "/floats/f64be") == ["1.0", "2.0", "3.0"])112        #expect(try values(file, "/floats/f16") == ["0.5", "-2.0"])113        #expect(try values(file, "/strings/fixed") == ["abc", "defghijk", ""])114        #expect(try values(file, "/strings/spacepad") == ["ab", "cde"])115        #expect(try values(file, "/strings/variable") == ["one", "deux é", ""])116        #expect(try values(file, "/enum/colors") == ["RED", "BLUE", "GREEN", "BLUE"])117        #expect(try values(file, "/array/vectors") == ["[1,2,3]", "[4,5,6]"])118        #expect(try values(file, "/scalar") == ["42.5"])119        let particles = try file.preview("/compound/particles")120        #expect(particles.memberNames == ["id", "mass", "name", "flags"])121        #expect(particles.values == ["1\t1.5\tp1\t1", "2\t2.25\tproton\t200", "-3\t0.0\t\t0"])122        let refs = try file.preview("/refs/objref")123        #expect(refs.values.isEmpty && refs.note == "The values cannot be shown: object and region references are not converted.")124    }125126    @Test func hyperslabs() throws {127        let temp = TempDir()128        let file = try SciFile(url: Fixture.everything(in: temp.url))129        // Rows 3, 8, 13 and columns 28, 29 of the chunked grid (crossing chunk boundaries).130        let p = try file.preview("/chunked/grid", start: [3, 28], count: [3, 2], stride: [5, 1])131        #expect(p.values == ["328", "329", "828", "829", "1328", "1329"])132        #expect(p.selection == SciSelection(start: [3, 28], count: [3, 2], stride: [5, 1]) && !p.isTruncated)133        // The default window starts at the origin and is cut to the element bound, as a rectangle.134        let head = try file.preview("/chunked/grid", maxElements: 60)135        #expect(head.selection.count == [2, 30] && head.isTruncated && head.values.first == "-999" && head.values.last == "129")136        // A window past the end is clipped; an invalid one is refused.137        #expect(try values(file, "/unlimited/series", start: [5], count: [10]) == ["7.5", "9.0"])138        #expect(try values(file, "/unlimited/series", start: [1], stride: [3]) == ["1.5", "6.0"])139        #expect(try values(file, "/unlimited/series", start: [99]) == [])140        #expect(throws: ConversionError.self) { _ = try file.preview("/chunked/grid", start: [0]) }141        #expect(throws: ConversionError.self) { _ = try file.backend.read("/chunked/grid", SciSelection(start: [19, 0], count: [2, 1], stride: [1, 1])) }142        #expect(throws: ConversionError.self) { _ = try file.backend.read("/chunked/grid", SciSelection(start: [0, 0], count: [1, 1], stride: [0, 1])) }143        #expect(throws: ConversionError.self) { _ = try file.preview("/ints") }144    }145146    @Test func linksAreReportedAndNeverFollowed() throws {147        let temp = TempDir()148        let file = try SciFile(url: Fixture.everything(in: temp.url))149        // Two independent defences: the library's external-link class is replaced by one that never traverses, and150        // every access property list the engine uses carries a callback that refuses the traversal.151        #expect(H5.sync { orix_h5_external_links_disabled() } == 1)152        let before = H5.sync { orix_h5_external_refusals() }153        let links = try file.children(of: "/links")154        let byName = Dictionary(uniqueKeysWithValues: links.map { ($0.name, $0.kind) })155        #expect(byName["ext"] == .externalLink(file: "other.h5", path: "/data"))156        #expect(byName["dangling"] == .danglingLink(target: "/nowhere"))157        #expect(byName["soft"] == .dataset && byName["alias"] == .dataset)158        #expect(links.first { $0.name == "soft" }?.softLinkTarget == "/scalar" && links.first { $0.name == "alias" }?.softLinkTarget == nil)159        #expect(file.notes.contains { $0.contains("/links/ext is a link to another file (other.h5)") })160        // The target file exists and holds [111, 222]; nothing reaches it, by any route.161        #expect(try SciFile(url: temp.file("other.h5")).preview("/data").values == ["111", "222"])162        #expect(throws: ConversionError.self) { _ = try file.backend.read("/links/ext", SciSelection(all: [2])) }163        #expect(throws: ConversionError.self) { _ = try file.node(at: "/links/ext/anything") }164        #expect(throws: ConversionError.self) { _ = try file.attributes(of: "/links/ext") }165        #expect(throws: ConversionError.self) { _ = try file.children(of: "/links/ext") }166        #expect(throws: ConversionError.self) { _ = try file.preview("/links/ext") }167        // Even a raw open with default property lists (which the engine never uses on a source) is refused.168        let h5 = try #require(file.backend as? H5Backend)169        H5.sync {170            #expect(H5Dopen2(h5.file, "/links/ext", 0) < 0)171            #expect(H5Oopen(h5.file, "/links/ext", 0) < 0)172            #expect(H5Dopen2(h5.file, "/links/ext", h5.dapl) < 0)173            _ = H5.lastError()174            #expect(orix_h5_external_refusals() >= before + 3)175        }176        // The soft link inside the file works; the hard-link alias is the same object.177        #expect(try values(file, "/links/soft") == ["42.5"])178        #expect(file.backend.identity(of: "/links/alias") == file.backend.identity(of: "/ints/i32"))179        #expect(file.backend.identity(of: "/links/alias") != file.backend.identity(of: "/ints/i64"))180    }181182    @Test func engineInspection() async throws {183        let temp = TempDir()184        let i = try await ScientificEngine().inspect(Fixture.everything(in: temp.url), format: .hdf5)185        func field(_ group: String, _ key: String) -> String? { i.fields.first { $0.group == group && $0.key == key }?.value }186        #expect(field("Scientific data", "Container") == "HDF5")187        #expect(field("Scientific data", "Groups") == "13" && field("Scientific data", "Datasets") == "28")188        #expect(field("Scientific data", "Links not followed") == "/links/dangling, /links/ext")189        #expect(field("Scientific data", "Dimensions") == "y = 20, x = 30, t = 5")190        #expect(field("Global attributes", "title") == "Everything" && field("Global attributes", "ratios") == "0.5, 1.5, 2.5")191        #expect(field("Datasets", "/chunked/grid") == "int16; shape y=20 × x=30; chunks 5 × 10; filters shuffle + deflate(6); fill -999; units “K”; standard_name air_temperature; packed (scale_factor 0.01, add_offset 273.15); 5 attribute(s)")192        #expect(field("Datasets", "/unlimited/series") == "float32; shape 7 (unlimited); chunks 4")193        #expect(field("Datasets", "/time/t") == "float64; shape t=5; coordinate variable; units “hours since 2000-01-01 00:00:00”; calendar standard; 2 attribute(s)")194        #expect(field("Datasets", "/scalar") == "float64; scalar; units “m”; 1 attribute(s)")195        #expect(i.entryCount == 28 && i.hasMetadata == true)196        #expect(i.notes.contains { $0.contains("external links are never followed") })197    }198}199200@Suite struct NetCDFInspectTests {201    @Test(arguments: [UInt8(1), 2, 5]) func classicFromIndependentBytes(_ version: UInt8) throws {202        let temp = TempDir()203        let file = try SciFile(url: Fixture.classic(temp.file("c\(version).nc"), version: version))204        #expect(file.formatDescription == ["", "NetCDF classic (CDF-1)", "NetCDF classic (CDF-2, 64-bit offset)", "", "", "NetCDF classic (CDF-5, 64-bit data)"][Int(version)])205        #expect(file.isNetCDF)206        let nodes = try file.children()207        #expect(nodes.map(\.name) == ["lat", "lon", "station", "time", "temp", "scalarv", "pres"] + (version == 5 ? ["big", "flags"] : []))208        let temp_ = try #require(nodes[4].dataset)209        #expect(temp_.type == .integer(bits: 16, signed: true))210        #expect(temp_.dimensions == [SciDimension(name: "time", length: 3, isUnlimited: true, coordinatePath: "/time", definitionPath: "/time"),211                                     SciDimension(name: "lat", length: 2, isUnlimited: false, coordinatePath: "/lat", definitionPath: "/lat"),212                                     SciDimension(name: "lon", length: 3, isUnlimited: false, coordinatePath: "/lon", definitionPath: "/lon")])213        #expect(temp_.fillValue == "-999")214        #expect(nodes[0].dataset?.isCoordinate == true && nodes[4].dataset?.isCoordinate == false)215        #expect(nodes[2].dataset?.type == .char && nodes[2].dataset?.shape == [2, 4])216        #expect(nodes[5].dataset?.shape == [])217        // Attributes.218        let global = try file.attributes(of: "/").items219        #expect(global.map(\.name) == ["title", "version"] && global[0].text() == "Classic" && global[1].text() == "7" && global[1].type == .integer(bits: 32, signed: true))220        let ta = try file.attributes(of: "/temp").items221        #expect(ta.map { "\($0.name)=\($0.text())" } == ["_FillValue=-999", "scale_factor=0.5", "units=K"])222        #expect(try file.conventions(of: "/time").units == "hours since 2020-01-01 00:00:00")223        // Values: fixed variables, record variables, hyperslabs across records.224        #expect(try values(file, "/lat") == ["45.0", "46.5"])225        #expect(try values(file, "/lon") == ["-75.0", "-74.5", "-74.0"])226        #expect(try values(file, "/scalarv") == ["3.5"])227        #expect(try values(file, "/time") == ["0", "6", "12"])228        #expect(try values(file, "/pres") == ["1000.5", "999.25", "998.0"])229        #expect(try values(file, "/station") == ["A", "B", "C", "D", "E", "F", "\0", "\0"])230        var expected: [String] = []231        for t in 0..<3 { for la in 0..<2 { for lo in 0..<3 { expected.append(String(Fixture.classicTemp(t, la, lo))) } } }232        #expect(try values(file, "/temp") == expected)233        #expect(try values(file, "/temp", start: [0, 1, 0], count: [3, 1, 2], stride: [2, 1, 2]) == ["10", "12", "210", "212"])234        #expect(try values(file, "/temp", start: [2, 0, 1], count: [1, 2, 1]) == ["201", "211"])235        #expect(try values(file, "/pres", start: [2]) == ["998.0"])236        if version == 5 {237            #expect(try values(file, "/big") == ["-9000000000", "9000000000"])238            #expect(try values(file, "/flags") == ["1", "200", "255"])239            #expect(nodes[7].dataset?.type == .integer(bits: 64, signed: true) && nodes[8].dataset?.type == .integer(bits: 8, signed: false))240            #expect(try file.attributes(of: "/big").items.first?.text() == "18446744073709551615")241        }242    }243244    @Test func netCDF4BuiltWithRawCalls() async throws {245        let temp = TempDir()246        let file = try SciFile(url: Fixture.netCDF4(temp.file("raw.nc")))247        #expect(file.formatDescription == "NetCDF-4 (HDF5)" && file.isNetCDF)248        // The dimension-only dataset "time" is a dimension, not a variable; creation order is kept.249        #expect(try file.children().map(\.name) == ["x", "temp", "label", "model"])250        let t = try #require(try file.node(at: "/temp").dataset)251        #expect(t.dimensions == [SciDimension(name: "time", length: 2, isUnlimited: true, coordinatePath: nil, definitionPath: "/time"),252                                 SciDimension(name: "x", length: 3, isUnlimited: false, coordinatePath: "/x", definitionPath: "/x")])253        #expect(t.filters.map(\.name) == ["deflate"] && t.chunkShape == [1, 3])254        #expect(try file.node(at: "/model/bias").dataset?.dimensions.first?.name == "x")255        // NetCDF-4 internals are not user attributes.256        #expect(try file.attributes(of: "/").items.map(\.name) == ["Conventions"])257        #expect(try file.attributes(of: "/x").items.map(\.name) == ["units"])258        #expect(try values(file, "/temp") == ["10.0", "11.0", "12.0", "20.0", "21.0", "22.0"])259        let i = try await ScientificEngine().inspect(file.url, format: .netcdf)260        #expect(i.fields.first { $0.key == "Dimensions" }?.value == "x = 3, time = 2 (unlimited)")261        #expect(i.fields.first { $0.key == "Container" }?.value == "NetCDF-4 (HDF5)")262    }263}264
Files main
Menu

Theme