Repository Public License
1// ORIX Convert — Packages/EngineScientific/Tests/EngineScientificTests/Fixtures.swift2// Description: Fixtures with known contents, generated at test time: an HDF5 file exercising every feature, written3// with the C library; a NetCDF-4-shaped HDF5 file built with raw dimension-scale calls; and NetCDF4// classic files (CDF-1, CDF-2, CDF-5) assembled byte by byte from the format specification — all5// independently of the engine's own writers.6// Author: Simon-Pierre Boucher — contact@spboucher.ai7import Foundation8import CHDF59import CHDF5Shim10@testable import EngineScientific1112enum Fixture {13 // MARK: HDF51415 static let i8: [Int8] = [-128, -1, 0, 127], u8: [UInt8] = [0, 1, 128, 255]16 static let i16: [Int16] = [-32768, -1, 0, 32767], u16: [UInt16] = [0, 1, 32768, 65535]17 static let i32: [Int32] = [Int32.min, -1, 0, Int32.max], u32: [UInt32] = [0, 1, 2_147_483_648, UInt32.max]18 static let i64: [Int64] = [Int64.min, -1, 0, Int64.max], u64: [UInt64] = [0, 1, 9_223_372_036_854_775_808, UInt64.max]19 static let f32: [Float] = [0.1, -1.5, 3.25, 1e-10, 1e20, .nan], f64: [Double] = [0.1, -1.5, 3.25, 1e-300, 1e300, .infinity]20 static let gridRows = 20, gridColumns = 3021 static func grid(_ r: Int, _ c: Int) -> Int16 { r == 0 && c == 0 ? -999 : Int16(r * 100 + c) }22 static func x(_ i: Int) -> Double { 10 + 0.5 * Double(i) }23 static func y(_ i: Int) -> Int32 { Int32(i * 2) }2425 /// Writes `everything.h5` (and `other.h5`, the target of its external link) into `dir`.26 @discardableResult27 static func everything(in dir: URL, name: String = "everything.h5") -> URL {28 let url = dir.appendingPathComponent(name)29 H5.sync {30 let other = C.create(dir.appendingPathComponent("other.h5"))31 C.numbers(other, "data", ORIX_T_INT32, [Int32(111), 222])32 H5Fclose(other)3334 let f = C.create(url)35 defer { H5Fclose(f) }36 // Root attributes: fixed string, scalar integer, array of doubles, variable-length strings.37 C.attribute(f, "title", text: "Everything")38 C.attribute(f, "version", ORIX_T_INT32, [Int32(3)], scalar: true)39 C.attribute(f, "ratios", ORIX_T_DOUBLE, [0.5, 1.5, 2.5])40 let vs = C.variableString()41 let tags: [UnsafeMutablePointer<CChar>?] = [strdup("alpha"), strdup("βeta")]42 tags.withUnsafeBytes { C.attribute(f, "tags", file: vs, memory: vs, dims: [2], data: $0.baseAddress!) }43 tags.forEach { free($0) }4445 let ints = C.group(f, "ints")46 C.attribute(ints, "note", text: "integers")47 C.numbers(ints, "i8", ORIX_T_INT8, i8); C.numbers(ints, "u8", ORIX_T_UINT8, u8)48 C.numbers(ints, "i16", ORIX_T_INT16, i16); C.numbers(ints, "u16", ORIX_T_UINT16, u16)49 C.numbers(ints, "i32", ORIX_T_INT32, i32); C.numbers(ints, "u32", ORIX_T_UINT32, u32)50 C.numbers(ints, "i64", ORIX_T_INT64, i64); C.numbers(ints, "u64", ORIX_T_UINT64, u64)51 H5Gclose(ints)5253 let floats = C.group(f, "floats")54 C.numbers(floats, "f32", ORIX_T_FLOAT, f32, dims: [2, 3])55 C.numbers(floats, "f64", ORIX_T_DOUBLE, f64, dims: [2, 3])56 let be = H5Tcopy(H5.type(ORIX_T_DOUBLE))57 H5Tset_order(be, H5T_ORDER_BE)58 C.numbers(floats, "f64be", ORIX_T_DOUBLE, [1.0, 2.0, 3.0], fileType: be)59 H5Tclose(be)60 C.numbers(floats, "f16", ORIX_T_FLOAT, [Float(0.5), -2.0], fileType: H5.type(ORIX_T_FLOAT16))61 H5Gclose(floats)6263 let strings = C.group(f, "strings")64 let fixed = C.stringType(8)65 C.packed(["abc", "defghijk", ""], size: 8).withUnsafeBytes { _ = C.dataset(strings, "fixed", file: fixed, memory: fixed, dims: [3], data: $0.baseAddress) }66 H5Tclose(fixed)67 let spaced = C.stringType(5, pad: H5T_STR_SPACEPAD)68 C.packed(["ab", "cde"], size: 5, pad: 0x20).withUnsafeBytes { _ = C.dataset(strings, "spacepad", file: spaced, memory: spaced, dims: [2], data: $0.baseAddress) }69 H5Tclose(spaced)70 let words: [UnsafeMutablePointer<CChar>?] = [strdup("one"), strdup("deux é"), strdup("")]71 words.withUnsafeBytes { _ = C.dataset(strings, "variable", file: vs, memory: vs, dims: [3], data: $0.baseAddress) }72 words.forEach { free($0) }73 H5Gclose(strings)7475 // Compound {int32 id; float64 mass; char name[6]; uint8 flags}, 19 bytes, packed.76 let compound = C.group(f, "compound")77 let ct = H5Tcreate(H5T_COMPOUND, 19)78 let name6 = C.stringType(6)79 H5Tinsert(ct, "id", 0, H5.type(ORIX_T_INT32)); H5Tinsert(ct, "mass", 4, H5.type(ORIX_T_DOUBLE))80 H5Tinsert(ct, "name", 12, name6); H5Tinsert(ct, "flags", 18, H5.type(ORIX_T_UINT8))81 var raw: [UInt8] = []82 for (id, mass, name, flags) in [(Int32(1), 1.5, "p1", UInt8(1)), (2, 2.25, "proton", 200), (-3, 0.0, "", 0)] {83 withUnsafeBytes(of: id) { raw += $0 }84 withUnsafeBytes(of: mass) { raw += $0 }85 raw += C.packed([name], size: 6)86 raw.append(flags)87 }88 raw.withUnsafeBytes { _ = C.dataset(compound, "particles", file: ct, memory: ct, dims: [3], data: $0.baseAddress) }89 H5Tclose(name6); H5Tclose(ct); H5Gclose(compound)9091 // Dimension scales and a chunked, shuffled, deflated grid with a fill value and CF attributes.92 let dims = C.group(f, "dims")93 let xs = C.numbers(dims, "x", ORIX_T_DOUBLE, (0..<gridColumns).map(x), close: false)94 let ys = C.numbers(dims, "y", ORIX_T_INT32, (0..<gridRows).map(y), close: false)95 H5DSset_scale(xs, "x"); H5DSset_scale(ys, "y")96 C.attribute(xs, "units", text: "km")97 let chunked = C.group(f, "chunked")98 let dcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))99 H5Pset_chunk(dcpl, 2, [hsize_t(5), 10])100 H5Pset_shuffle(dcpl); H5Pset_deflate(dcpl, 6)101 var fill = Int16(-999)102 H5Pset_fill_value(dcpl, H5.type(ORIX_T_INT16), &fill)103 var values: [Int16] = []104 for r in 0..<gridRows { for c in 0..<gridColumns { values.append(grid(r, c)) } }105 let g = C.numbers(chunked, "grid", ORIX_T_INT16, values, dims: [hsize_t(gridRows), hsize_t(gridColumns)], dcpl: dcpl, close: false)106 H5Pclose(dcpl)107 C.attribute(g, "units", text: "K")108 C.attribute(g, "standard_name", text: "air_temperature")109 C.attribute(g, "scale_factor", ORIX_T_DOUBLE, [0.01])110 C.attribute(g, "add_offset", ORIX_T_DOUBLE, [273.15])111 C.attribute(g, "_FillValue", ORIX_T_INT16, [Int16(-999)])112 H5DSattach_scale(g, ys, 0); H5DSattach_scale(g, xs, 1)113 H5Dclose(g); H5Dclose(xs); H5Dclose(ys); H5Gclose(dims); H5Gclose(chunked)114115 // Unlimited dimension.116 let unlimited = C.group(f, "unlimited")117 let ucpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))118 H5Pset_chunk(ucpl, 1, [hsize_t(4)])119 C.numbers(unlimited, "series", ORIX_T_FLOAT, (0..<7).map { Float($0) * 1.5 }, dims: [7], maxDims: [H5.unlimited], dcpl: ucpl)120 H5Pclose(ucpl); H5Gclose(unlimited)121122 // Scalar, enumeration, array type.123 var scalar = 42.5124 let s = C.dataset(f, "scalar", file: H5.type(ORIX_T_DOUBLE), memory: H5.type(ORIX_T_DOUBLE), dims: nil, data: &scalar, close: false)125 C.attribute(s, "units", text: "m")126 H5Dclose(s)127 let e = C.group(f, "enum")128 let et = H5Tenum_create(H5.type(ORIX_T_INT8))129 for (name, value) in [("RED", Int8(0)), ("GREEN", 1), ("BLUE", 2)] { var v = value; H5Tenum_insert(et, name, &v) }130 [Int8(0), 2, 1, 2].withUnsafeBytes { _ = C.dataset(e, "colors", file: et, memory: et, dims: [4], data: $0.baseAddress) }131 H5Tclose(et); H5Gclose(e)132 let a = C.group(f, "array")133 let at = H5Tarray_create2(H5.type(ORIX_T_INT32), 1, [hsize_t(3)])134 [Int32(1), 2, 3, 4, 5, 6].withUnsafeBytes { _ = C.dataset(a, "vectors", file: at, memory: at, dims: [2], data: $0.baseAddress) }135 H5Tclose(at); H5Gclose(a)136137 // Time axis with a fill value in the data.138 let time = C.group(f, "time")139 let t = C.numbers(time, "t", ORIX_T_DOUBLE, [0, 1.5, 24, 48, 100_000], close: false)140 H5DSset_scale(t, "t")141 C.attribute(t, "units", text: "hours since 2000-01-01 00:00:00")142 C.attribute(t, "calendar", text: "standard")143 let tv = C.numbers(time, "values", ORIX_T_INT32, [Int32(1), 2, -9999, 4, 5], close: false)144 C.attribute(tv, "_FillValue", ORIX_T_INT32, [Int32(-9999)])145 H5DSattach_scale(tv, t, 0)146 H5Dclose(t); H5Dclose(tv); H5Gclose(time)147148 // Things that must be reported, skipped or refused.149 let refs = C.group(f, "refs")150 var ref = H5R_ref_t()151 precondition(H5Rcreate_object(f, "/scalar", 0, &ref) >= 0)152 _ = C.dataset(refs, "objref", file: H5.type(ORIX_T_STD_REF), memory: H5.type(ORIX_T_STD_REF), dims: [1], data: &ref)153 H5Rdestroy(&ref)154 H5Gclose(refs)155 let links = C.group(f, "links")156 H5Lcreate_soft("/scalar", links, "soft", 0, 0)157 H5Lcreate_soft("/nowhere", links, "dangling", 0, 0)158 H5Lcreate_external("other.h5", "/data", links, "ext", 0, 0)159 H5Lcreate_hard(f, "/ints/i32", links, "alias", 0, 0)160 H5Gclose(links)161 H5Gclose(C.group(f, "empty"))162 }163 return url164 }165166 /// A small HDF5 file with a single two-dimensional dataset and nothing else.167 @discardableResult168 static func single(_ url: URL, rows: Int = 3, columns: Int = 4) -> URL {169 H5.sync {170 let f = C.create(url)171 C.numbers(f, "m", ORIX_T_INT32, (0..<(rows * columns)).map { Int32($0) }, dims: [hsize_t(rows), hsize_t(columns)])172 H5Fclose(f)173 }174 return url175 }176177 /// A NetCDF-4 file built with raw HDF5 calls, following the NetCDF-4 file format rules: creation-order tracking,178 /// `_NCProperties`, an unlimited dimension `time` without coordinate variable (placeholder dataset), a179 /// coordinate variable `x`, and `temp(time, x)` attached to both.180 @discardableResult181 static func netCDF4(_ url: URL) -> URL {182 H5.sync {183 let fcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_FILE_CREATE))184 H5Pset_link_creation_order(fcpl, C.orderTracked)185 H5Pset_attr_creation_order(fcpl, C.orderTracked)186 let fapl = orix_h5_fapl(1)187 let f = H5Fcreate(url.path, UInt32(H5F_ACC_TRUNC), fcpl, fapl)188 precondition(f >= 0)189 H5Pclose(fcpl); H5Pclose(fapl)190 C.attribute(f, "_NCProperties", text: "version=2,netcdf=4.9.2,hdf5=1.14.3")191 C.attribute(f, "Conventions", text: "CF-1.8")192 // Dimension "time": unlimited, currently 2, no coordinate variable.193 let tcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))194 H5Pset_chunk(tcpl, 1, [hsize_t(1)])195 let time = C.dataset(f, "time", file: H5.type(ORIX_T_IEEE_F32BE), memory: H5.type(ORIX_T_FLOAT), dims: [2], maxDims: [H5.unlimited], dcpl: tcpl, data: nil, close: false)196 H5Pclose(tcpl)197 H5DSset_scale(time, "This is a netCDF dimension but not a netCDF variable. 2")198 C.attribute(time, "_Netcdf4Dimid", ORIX_T_INT32, [Int32(0)], scalar: true)199 // Coordinate variable "x".200 let x = C.numbers(f, "x", ORIX_T_FLOAT, [Float(1.5), 2.5, 3.5], close: false)201 H5DSset_scale(x, "x")202 C.attribute(x, "_Netcdf4Dimid", ORIX_T_INT32, [Int32(1)], scalar: true)203 C.attribute(x, "units", text: "m")204 // temp(time, x).205 let dcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))206 H5Pset_chunk(dcpl, 2, [hsize_t(1), 3])207 H5Pset_deflate(dcpl, 2)208 let temp = C.numbers(f, "temp", ORIX_T_FLOAT, [Float(10), 11, 12, 20, 21, 22], dims: [2, 3], maxDims: [H5.unlimited, 3], dcpl: dcpl, close: false)209 H5Pclose(dcpl)210 C.attribute(temp, "units", text: "degC")211 H5DSattach_scale(temp, time, 0); H5DSattach_scale(temp, x, 1)212 // A character variable: station names as characters along "x".213 let c1 = C.stringType(1, pad: H5T_STR_NULLTERM)214 Array("abc".utf8).withUnsafeBytes { p in215 let label = C.dataset(f, "label", file: c1, memory: c1, dims: [3], data: p.baseAddress, close: false)216 H5DSattach_scale(label, x, 0)217 H5Dclose(label)218 }219 H5Tclose(c1)220 // A subgroup with its own variable on the root dimension x.221 let g = C.group(f, "model")222 let inner = C.numbers(g, "bias", ORIX_T_DOUBLE, [0.1, 0.2, 0.3], close: false)223 H5DSattach_scale(inner, x, 0)224 H5Dclose(inner); H5Gclose(g)225 H5Dclose(temp); H5Dclose(x); H5Dclose(time); H5Fclose(f)226 }227 return url228 }229230 // MARK: NetCDF classic, byte by byte231232 /// Big-endian byte assembler for the classic header grammar.233 struct NCBytes {234 var data = [UInt8]()235 /// CDF-5 widens counts and sizes to 64 bits.236 let wide: Bool237 mutating func u32(_ v: UInt32) { data += withUnsafeBytes(of: v.bigEndian) { Array($0) } }238 mutating func u64(_ v: UInt64) { data += withUnsafeBytes(of: v.bigEndian) { Array($0) } }239 mutating func count(_ v: Int) { if wide { u64(UInt64(v)) } else { u32(UInt32(v)) } }240 mutating func pad() { while data.count % 4 != 0 { data.append(0) } }241 mutating func name(_ s: String) { count(s.utf8.count); data += Array(s.utf8); pad() }242 mutating func text(_ name: String, _ value: String) { self.name(name); u32(2); count(value.utf8.count); data += Array(value.utf8); pad() }243 mutating func attribute(_ name: String, type: UInt32, values: [UInt8], count n: Int) { self.name(name); u32(type); count(n); data += values; pad() }244 }245246 static func be<T: FixedWidthInteger>(_ v: T) -> [UInt8] { withUnsafeBytes(of: v.bigEndian) { Array($0) } }247 static func be(_ v: Float) -> [UInt8] { be(v.bitPattern) }248 static func be(_ v: Double) -> [UInt8] { be(v.bitPattern) }249250 static let classicRecords = 3251 static func classicTemp(_ t: Int, _ la: Int, _ lo: Int) -> Int16 { t == 1 && la == 0 && lo == 2 ? -999 : Int16(t * 100 + la * 10 + lo) }252253 /// A classic file with a record dimension and six variables.254 ///255 /// dimensions: time = UNLIMITED (3 currently), lat = 2, lon = 3, strlen = 4256 /// variables: float lat(lat); double lon(lon); char station(lat, strlen); int time(time);257 /// short temp(time, lat, lon); double scalarv; float pres(time)258 /// (CDF-5 only) int64 big(lat); ubyte flags(lon)259 static func classic(_ url: URL, version: UInt8 = 1) -> URL {260 let wide = version == 5261 func header(_ begins: [UInt64]) -> [UInt8] {262 var h = NCBytes(wide: wide)263 h.data = [0x43, 0x44, 0x46, version]264 h.count(classicRecords)265 h.u32(0x0A); h.count(4)266 h.name("time"); h.count(0)267 h.name("lat"); h.count(2)268 h.name("lon"); h.count(3)269 h.name("strlen"); h.count(4)270 h.u32(0x0C); h.count(2)271 h.text("title", "Classic")272 h.attribute("version", type: 4, values: be(Int32(7)), count: 1)273 h.u32(0x0B); h.count(wide ? 9 : 7)274 var index = 0275 func variable(_ name: String, dims: [Int], type: UInt32, vsize: Int, attributes: (inout NCBytes) -> Int) {276 h.name(name)277 h.count(dims.count)278 for d in dims { h.count(d) }279 var scratch = NCBytes(wide: wide)280 let n = attributes(&scratch)281 if n == 0 { h.u32(0); h.count(0) } else { h.u32(0x0C); h.count(n); h.data += scratch.data }282 h.u32(type)283 h.count(vsize)284 if version == 1 { h.u32(UInt32(begins[index])) } else { h.u64(begins[index]) }285 index += 1286 }287 variable("lat", dims: [1], type: 5, vsize: 8) { $0.text("units", "degrees_north"); return 1 }288 variable("lon", dims: [2], type: 6, vsize: 24) { _ in 0 }289 variable("station", dims: [1, 3], type: 2, vsize: 8) { _ in 0 }290 variable("time", dims: [0], type: 4, vsize: 4) { $0.text("units", "hours since 2020-01-01 00:00:00"); return 1 }291 variable("temp", dims: [0, 1, 2], type: 3, vsize: 12) {292 $0.attribute("_FillValue", type: 3, values: be(Int16(-999)), count: 1)293 $0.attribute("scale_factor", type: 6, values: be(0.5), count: 1)294 $0.text("units", "K")295 return 3296 }297 variable("scalarv", dims: [], type: 6, vsize: 8) { _ in 0 }298 variable("pres", dims: [0], type: 5, vsize: 4) { _ in 0 }299 if wide {300 variable("big", dims: [1], type: 10, vsize: 16) { $0.attribute("huge", type: 11, values: be(UInt64.max), count: 1); return 1 }301 variable("flags", dims: [2], type: 7, vsize: 4) { _ in 0 }302 }303 return h.data304 }305 // Layout: fixed variables in order (each padded to 4 bytes), then the records.306 let fixedSizes: [(Int, Int)] = [(0, 8), (1, 24), (2, 8), (5, 8)] + (wide ? [(7, 16), (8, 4)] : [])307 var begins = [UInt64](repeating: 0, count: wide ? 9 : 7)308 var cursor = UInt64(header(begins).count)309 for (i, size) in fixedSizes { begins[i] = cursor; cursor += UInt64(size) }310 let recordStart = cursor311 begins[3] = recordStart; begins[4] = recordStart + 4; begins[6] = recordStart + 16312 var bytes = header(begins)313 bytes += be(Float(45.0)) + be(Float(46.5))314 bytes += be(-75.0) + be(-74.5) + be(-74.0)315 bytes += Array("ABCD".utf8) + Array("EF".utf8) + [0, 0]316 bytes += be(3.5)317 if wide {318 bytes += be(Int64(-9_000_000_000)) + be(Int64(9_000_000_000))319 bytes += [1, 200, 255, 0]320 }321 precondition(UInt64(bytes.count) == recordStart)322 for t in 0..<classicRecords {323 bytes += be(Int32(t * 6))324 for la in 0..<2 { for lo in 0..<3 { bytes += be(classicTemp(t, la, lo)) } }325 bytes += be([Float(1000.5), 999.25, 998.0][t])326 }327 try? Data(bytes).write(to: url)328 return url329 }330}331