SPB Git

Repository Public License

Fixtures.swift 18.5 KB 331 lines swift

1// ORIX Convert — Packages/EngineScientific/Tests/EngineScientificTests/Fixtures.swift2// Description: Fixtures with known contents, generated at test time: an HDF5 file exercising every feature, written3//              with the C library; a NetCDF-4-shaped HDF5 file built with raw dimension-scale calls; and NetCDF4//              classic files (CDF-1, CDF-2, CDF-5) assembled byte by byte from the format specification — all5//              independently of the engine's own writers.6// Author: Simon-Pierre Boucher — contact@spboucher.ai7import Foundation8import CHDF59import CHDF5Shim10@testable import EngineScientific1112enum Fixture {13    // MARK: HDF51415    static let i8: [Int8] = [-128, -1, 0, 127], u8: [UInt8] = [0, 1, 128, 255]16    static let i16: [Int16] = [-32768, -1, 0, 32767], u16: [UInt16] = [0, 1, 32768, 65535]17    static let i32: [Int32] = [Int32.min, -1, 0, Int32.max], u32: [UInt32] = [0, 1, 2_147_483_648, UInt32.max]18    static let i64: [Int64] = [Int64.min, -1, 0, Int64.max], u64: [UInt64] = [0, 1, 9_223_372_036_854_775_808, UInt64.max]19    static let f32: [Float] = [0.1, -1.5, 3.25, 1e-10, 1e20, .nan], f64: [Double] = [0.1, -1.5, 3.25, 1e-300, 1e300, .infinity]20    static let gridRows = 20, gridColumns = 3021    static func grid(_ r: Int, _ c: Int) -> Int16 { r == 0 && c == 0 ? -999 : Int16(r * 100 + c) }22    static func x(_ i: Int) -> Double { 10 + 0.5 * Double(i) }23    static func y(_ i: Int) -> Int32 { Int32(i * 2) }2425    /// Writes `everything.h5` (and `other.h5`, the target of its external link) into `dir`.26    @discardableResult27    static func everything(in dir: URL, name: String = "everything.h5") -> URL {28        let url = dir.appendingPathComponent(name)29        H5.sync {30            let other = C.create(dir.appendingPathComponent("other.h5"))31            C.numbers(other, "data", ORIX_T_INT32, [Int32(111), 222])32            H5Fclose(other)3334            let f = C.create(url)35            defer { H5Fclose(f) }36            // Root attributes: fixed string, scalar integer, array of doubles, variable-length strings.37            C.attribute(f, "title", text: "Everything")38            C.attribute(f, "version", ORIX_T_INT32, [Int32(3)], scalar: true)39            C.attribute(f, "ratios", ORIX_T_DOUBLE, [0.5, 1.5, 2.5])40            let vs = C.variableString()41            let tags: [UnsafeMutablePointer<CChar>?] = [strdup("alpha"), strdup("βeta")]42            tags.withUnsafeBytes { C.attribute(f, "tags", file: vs, memory: vs, dims: [2], data: $0.baseAddress!) }43            tags.forEach { free($0) }4445            let ints = C.group(f, "ints")46            C.attribute(ints, "note", text: "integers")47            C.numbers(ints, "i8", ORIX_T_INT8, i8); C.numbers(ints, "u8", ORIX_T_UINT8, u8)48            C.numbers(ints, "i16", ORIX_T_INT16, i16); C.numbers(ints, "u16", ORIX_T_UINT16, u16)49            C.numbers(ints, "i32", ORIX_T_INT32, i32); C.numbers(ints, "u32", ORIX_T_UINT32, u32)50            C.numbers(ints, "i64", ORIX_T_INT64, i64); C.numbers(ints, "u64", ORIX_T_UINT64, u64)51            H5Gclose(ints)5253            let floats = C.group(f, "floats")54            C.numbers(floats, "f32", ORIX_T_FLOAT, f32, dims: [2, 3])55            C.numbers(floats, "f64", ORIX_T_DOUBLE, f64, dims: [2, 3])56            let be = H5Tcopy(H5.type(ORIX_T_DOUBLE))57            H5Tset_order(be, H5T_ORDER_BE)58            C.numbers(floats, "f64be", ORIX_T_DOUBLE, [1.0, 2.0, 3.0], fileType: be)59            H5Tclose(be)60            C.numbers(floats, "f16", ORIX_T_FLOAT, [Float(0.5), -2.0], fileType: H5.type(ORIX_T_FLOAT16))61            H5Gclose(floats)6263            let strings = C.group(f, "strings")64            let fixed = C.stringType(8)65            C.packed(["abc", "defghijk", ""], size: 8).withUnsafeBytes { _ = C.dataset(strings, "fixed", file: fixed, memory: fixed, dims: [3], data: $0.baseAddress) }66            H5Tclose(fixed)67            let spaced = C.stringType(5, pad: H5T_STR_SPACEPAD)68            C.packed(["ab", "cde"], size: 5, pad: 0x20).withUnsafeBytes { _ = C.dataset(strings, "spacepad", file: spaced, memory: spaced, dims: [2], data: $0.baseAddress) }69            H5Tclose(spaced)70            let words: [UnsafeMutablePointer<CChar>?] = [strdup("one"), strdup("deux é"), strdup("")]71            words.withUnsafeBytes { _ = C.dataset(strings, "variable", file: vs, memory: vs, dims: [3], data: $0.baseAddress) }72            words.forEach { free($0) }73            H5Gclose(strings)7475            // Compound {int32 id; float64 mass; char name[6]; uint8 flags}, 19 bytes, packed.76            let compound = C.group(f, "compound")77            let ct = H5Tcreate(H5T_COMPOUND, 19)78            let name6 = C.stringType(6)79            H5Tinsert(ct, "id", 0, H5.type(ORIX_T_INT32)); H5Tinsert(ct, "mass", 4, H5.type(ORIX_T_DOUBLE))80            H5Tinsert(ct, "name", 12, name6); H5Tinsert(ct, "flags", 18, H5.type(ORIX_T_UINT8))81            var raw: [UInt8] = []82            for (id, mass, name, flags) in [(Int32(1), 1.5, "p1", UInt8(1)), (2, 2.25, "proton", 200), (-3, 0.0, "", 0)] {83                withUnsafeBytes(of: id) { raw += $0 }84                withUnsafeBytes(of: mass) { raw += $0 }85                raw += C.packed([name], size: 6)86                raw.append(flags)87            }88            raw.withUnsafeBytes { _ = C.dataset(compound, "particles", file: ct, memory: ct, dims: [3], data: $0.baseAddress) }89            H5Tclose(name6); H5Tclose(ct); H5Gclose(compound)9091            // Dimension scales and a chunked, shuffled, deflated grid with a fill value and CF attributes.92            let dims = C.group(f, "dims")93            let xs = C.numbers(dims, "x", ORIX_T_DOUBLE, (0..<gridColumns).map(x), close: false)94            let ys = C.numbers(dims, "y", ORIX_T_INT32, (0..<gridRows).map(y), close: false)95            H5DSset_scale(xs, "x"); H5DSset_scale(ys, "y")96            C.attribute(xs, "units", text: "km")97            let chunked = C.group(f, "chunked")98            let dcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))99            H5Pset_chunk(dcpl, 2, [hsize_t(5), 10])100            H5Pset_shuffle(dcpl); H5Pset_deflate(dcpl, 6)101            var fill = Int16(-999)102            H5Pset_fill_value(dcpl, H5.type(ORIX_T_INT16), &fill)103            var values: [Int16] = []104            for r in 0..<gridRows { for c in 0..<gridColumns { values.append(grid(r, c)) } }105            let g = C.numbers(chunked, "grid", ORIX_T_INT16, values, dims: [hsize_t(gridRows), hsize_t(gridColumns)], dcpl: dcpl, close: false)106            H5Pclose(dcpl)107            C.attribute(g, "units", text: "K")108            C.attribute(g, "standard_name", text: "air_temperature")109            C.attribute(g, "scale_factor", ORIX_T_DOUBLE, [0.01])110            C.attribute(g, "add_offset", ORIX_T_DOUBLE, [273.15])111            C.attribute(g, "_FillValue", ORIX_T_INT16, [Int16(-999)])112            H5DSattach_scale(g, ys, 0); H5DSattach_scale(g, xs, 1)113            H5Dclose(g); H5Dclose(xs); H5Dclose(ys); H5Gclose(dims); H5Gclose(chunked)114115            // Unlimited dimension.116            let unlimited = C.group(f, "unlimited")117            let ucpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))118            H5Pset_chunk(ucpl, 1, [hsize_t(4)])119            C.numbers(unlimited, "series", ORIX_T_FLOAT, (0..<7).map { Float($0) * 1.5 }, dims: [7], maxDims: [H5.unlimited], dcpl: ucpl)120            H5Pclose(ucpl); H5Gclose(unlimited)121122            // Scalar, enumeration, array type.123            var scalar = 42.5124            let s = C.dataset(f, "scalar", file: H5.type(ORIX_T_DOUBLE), memory: H5.type(ORIX_T_DOUBLE), dims: nil, data: &scalar, close: false)125            C.attribute(s, "units", text: "m")126            H5Dclose(s)127            let e = C.group(f, "enum")128            let et = H5Tenum_create(H5.type(ORIX_T_INT8))129            for (name, value) in [("RED", Int8(0)), ("GREEN", 1), ("BLUE", 2)] { var v = value; H5Tenum_insert(et, name, &v) }130            [Int8(0), 2, 1, 2].withUnsafeBytes { _ = C.dataset(e, "colors", file: et, memory: et, dims: [4], data: $0.baseAddress) }131            H5Tclose(et); H5Gclose(e)132            let a = C.group(f, "array")133            let at = H5Tarray_create2(H5.type(ORIX_T_INT32), 1, [hsize_t(3)])134            [Int32(1), 2, 3, 4, 5, 6].withUnsafeBytes { _ = C.dataset(a, "vectors", file: at, memory: at, dims: [2], data: $0.baseAddress) }135            H5Tclose(at); H5Gclose(a)136137            // Time axis with a fill value in the data.138            let time = C.group(f, "time")139            let t = C.numbers(time, "t", ORIX_T_DOUBLE, [0, 1.5, 24, 48, 100_000], close: false)140            H5DSset_scale(t, "t")141            C.attribute(t, "units", text: "hours since 2000-01-01 00:00:00")142            C.attribute(t, "calendar", text: "standard")143            let tv = C.numbers(time, "values", ORIX_T_INT32, [Int32(1), 2, -9999, 4, 5], close: false)144            C.attribute(tv, "_FillValue", ORIX_T_INT32, [Int32(-9999)])145            H5DSattach_scale(tv, t, 0)146            H5Dclose(t); H5Dclose(tv); H5Gclose(time)147148            // Things that must be reported, skipped or refused.149            let refs = C.group(f, "refs")150            var ref = H5R_ref_t()151            precondition(H5Rcreate_object(f, "/scalar", 0, &ref) >= 0)152            _ = C.dataset(refs, "objref", file: H5.type(ORIX_T_STD_REF), memory: H5.type(ORIX_T_STD_REF), dims: [1], data: &ref)153            H5Rdestroy(&ref)154            H5Gclose(refs)155            let links = C.group(f, "links")156            H5Lcreate_soft("/scalar", links, "soft", 0, 0)157            H5Lcreate_soft("/nowhere", links, "dangling", 0, 0)158            H5Lcreate_external("other.h5", "/data", links, "ext", 0, 0)159            H5Lcreate_hard(f, "/ints/i32", links, "alias", 0, 0)160            H5Gclose(links)161            H5Gclose(C.group(f, "empty"))162        }163        return url164    }165166    /// A small HDF5 file with a single two-dimensional dataset and nothing else.167    @discardableResult168    static func single(_ url: URL, rows: Int = 3, columns: Int = 4) -> URL {169        H5.sync {170            let f = C.create(url)171            C.numbers(f, "m", ORIX_T_INT32, (0..<(rows * columns)).map { Int32($0) }, dims: [hsize_t(rows), hsize_t(columns)])172            H5Fclose(f)173        }174        return url175    }176177    /// A NetCDF-4 file built with raw HDF5 calls, following the NetCDF-4 file format rules: creation-order tracking,178    /// `_NCProperties`, an unlimited dimension `time` without coordinate variable (placeholder dataset), a179    /// coordinate variable `x`, and `temp(time, x)` attached to both.180    @discardableResult181    static func netCDF4(_ url: URL) -> URL {182        H5.sync {183            let fcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_FILE_CREATE))184            H5Pset_link_creation_order(fcpl, C.orderTracked)185            H5Pset_attr_creation_order(fcpl, C.orderTracked)186            let fapl = orix_h5_fapl(1)187            let f = H5Fcreate(url.path, UInt32(H5F_ACC_TRUNC), fcpl, fapl)188            precondition(f >= 0)189            H5Pclose(fcpl); H5Pclose(fapl)190            C.attribute(f, "_NCProperties", text: "version=2,netcdf=4.9.2,hdf5=1.14.3")191            C.attribute(f, "Conventions", text: "CF-1.8")192            // Dimension "time": unlimited, currently 2, no coordinate variable.193            let tcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))194            H5Pset_chunk(tcpl, 1, [hsize_t(1)])195            let time = C.dataset(f, "time", file: H5.type(ORIX_T_IEEE_F32BE), memory: H5.type(ORIX_T_FLOAT), dims: [2], maxDims: [H5.unlimited], dcpl: tcpl, data: nil, close: false)196            H5Pclose(tcpl)197            H5DSset_scale(time, "This is a netCDF dimension but not a netCDF variable.         2")198            C.attribute(time, "_Netcdf4Dimid", ORIX_T_INT32, [Int32(0)], scalar: true)199            // Coordinate variable "x".200            let x = C.numbers(f, "x", ORIX_T_FLOAT, [Float(1.5), 2.5, 3.5], close: false)201            H5DSset_scale(x, "x")202            C.attribute(x, "_Netcdf4Dimid", ORIX_T_INT32, [Int32(1)], scalar: true)203            C.attribute(x, "units", text: "m")204            // temp(time, x).205            let dcpl = H5Pcreate(orix_h5_plist_class(ORIX_P_DATASET_CREATE))206            H5Pset_chunk(dcpl, 2, [hsize_t(1), 3])207            H5Pset_deflate(dcpl, 2)208            let temp = C.numbers(f, "temp", ORIX_T_FLOAT, [Float(10), 11, 12, 20, 21, 22], dims: [2, 3], maxDims: [H5.unlimited, 3], dcpl: dcpl, close: false)209            H5Pclose(dcpl)210            C.attribute(temp, "units", text: "degC")211            H5DSattach_scale(temp, time, 0); H5DSattach_scale(temp, x, 1)212            // A character variable: station names as characters along "x".213            let c1 = C.stringType(1, pad: H5T_STR_NULLTERM)214            Array("abc".utf8).withUnsafeBytes { p in215                let label = C.dataset(f, "label", file: c1, memory: c1, dims: [3], data: p.baseAddress, close: false)216                H5DSattach_scale(label, x, 0)217                H5Dclose(label)218            }219            H5Tclose(c1)220            // A subgroup with its own variable on the root dimension x.221            let g = C.group(f, "model")222            let inner = C.numbers(g, "bias", ORIX_T_DOUBLE, [0.1, 0.2, 0.3], close: false)223            H5DSattach_scale(inner, x, 0)224            H5Dclose(inner); H5Gclose(g)225            H5Dclose(temp); H5Dclose(x); H5Dclose(time); H5Fclose(f)226        }227        return url228    }229230    // MARK: NetCDF classic, byte by byte231232    /// Big-endian byte assembler for the classic header grammar.233    struct NCBytes {234        var data = [UInt8]()235        /// CDF-5 widens counts and sizes to 64 bits.236        let wide: Bool237        mutating func u32(_ v: UInt32) { data += withUnsafeBytes(of: v.bigEndian) { Array($0) } }238        mutating func u64(_ v: UInt64) { data += withUnsafeBytes(of: v.bigEndian) { Array($0) } }239        mutating func count(_ v: Int) { if wide { u64(UInt64(v)) } else { u32(UInt32(v)) } }240        mutating func pad() { while data.count % 4 != 0 { data.append(0) } }241        mutating func name(_ s: String) { count(s.utf8.count); data += Array(s.utf8); pad() }242        mutating func text(_ name: String, _ value: String) { self.name(name); u32(2); count(value.utf8.count); data += Array(value.utf8); pad() }243        mutating func attribute(_ name: String, type: UInt32, values: [UInt8], count n: Int) { self.name(name); u32(type); count(n); data += values; pad() }244    }245246    static func be<T: FixedWidthInteger>(_ v: T) -> [UInt8] { withUnsafeBytes(of: v.bigEndian) { Array($0) } }247    static func be(_ v: Float) -> [UInt8] { be(v.bitPattern) }248    static func be(_ v: Double) -> [UInt8] { be(v.bitPattern) }249250    static let classicRecords = 3251    static func classicTemp(_ t: Int, _ la: Int, _ lo: Int) -> Int16 { t == 1 && la == 0 && lo == 2 ? -999 : Int16(t * 100 + la * 10 + lo) }252253    /// A classic file with a record dimension and six variables.254    ///255    ///     dimensions: time = UNLIMITED (3 currently), lat = 2, lon = 3, strlen = 4256    ///     variables:  float lat(lat); double lon(lon); char station(lat, strlen); int time(time);257    ///                 short temp(time, lat, lon); double scalarv; float pres(time)258    ///                 (CDF-5 only) int64 big(lat); ubyte flags(lon)259    static func classic(_ url: URL, version: UInt8 = 1) -> URL {260        let wide = version == 5261        func header(_ begins: [UInt64]) -> [UInt8] {262            var h = NCBytes(wide: wide)263            h.data = [0x43, 0x44, 0x46, version]264            h.count(classicRecords)265            h.u32(0x0A); h.count(4)266            h.name("time"); h.count(0)267            h.name("lat"); h.count(2)268            h.name("lon"); h.count(3)269            h.name("strlen"); h.count(4)270            h.u32(0x0C); h.count(2)271            h.text("title", "Classic")272            h.attribute("version", type: 4, values: be(Int32(7)), count: 1)273            h.u32(0x0B); h.count(wide ? 9 : 7)274            var index = 0275            func variable(_ name: String, dims: [Int], type: UInt32, vsize: Int, attributes: (inout NCBytes) -> Int) {276                h.name(name)277                h.count(dims.count)278                for d in dims { h.count(d) }279                var scratch = NCBytes(wide: wide)280                let n = attributes(&scratch)281                if n == 0 { h.u32(0); h.count(0) } else { h.u32(0x0C); h.count(n); h.data += scratch.data }282                h.u32(type)283                h.count(vsize)284                if version == 1 { h.u32(UInt32(begins[index])) } else { h.u64(begins[index]) }285                index += 1286            }287            variable("lat", dims: [1], type: 5, vsize: 8) { $0.text("units", "degrees_north"); return 1 }288            variable("lon", dims: [2], type: 6, vsize: 24) { _ in 0 }289            variable("station", dims: [1, 3], type: 2, vsize: 8) { _ in 0 }290            variable("time", dims: [0], type: 4, vsize: 4) { $0.text("units", "hours since 2020-01-01 00:00:00"); return 1 }291            variable("temp", dims: [0, 1, 2], type: 3, vsize: 12) {292                $0.attribute("_FillValue", type: 3, values: be(Int16(-999)), count: 1)293                $0.attribute("scale_factor", type: 6, values: be(0.5), count: 1)294                $0.text("units", "K")295                return 3296            }297            variable("scalarv", dims: [], type: 6, vsize: 8) { _ in 0 }298            variable("pres", dims: [0], type: 5, vsize: 4) { _ in 0 }299            if wide {300                variable("big", dims: [1], type: 10, vsize: 16) { $0.attribute("huge", type: 11, values: be(UInt64.max), count: 1); return 1 }301                variable("flags", dims: [2], type: 7, vsize: 4) { _ in 0 }302            }303            return h.data304        }305        // Layout: fixed variables in order (each padded to 4 bytes), then the records.306        let fixedSizes: [(Int, Int)] = [(0, 8), (1, 24), (2, 8), (5, 8)] + (wide ? [(7, 16), (8, 4)] : [])307        var begins = [UInt64](repeating: 0, count: wide ? 9 : 7)308        var cursor = UInt64(header(begins).count)309        for (i, size) in fixedSizes { begins[i] = cursor; cursor += UInt64(size) }310        let recordStart = cursor311        begins[3] = recordStart; begins[4] = recordStart + 4; begins[6] = recordStart + 16312        var bytes = header(begins)313        bytes += be(Float(45.0)) + be(Float(46.5))314        bytes += be(-75.0) + be(-74.5) + be(-74.0)315        bytes += Array("ABCD".utf8) + Array("EF".utf8) + [0, 0]316        bytes += be(3.5)317        if wide {318            bytes += be(Int64(-9_000_000_000)) + be(Int64(9_000_000_000))319            bytes += [1, 200, 255, 0]320        }321        precondition(UInt64(bytes.count) == recordStart)322        for t in 0..<classicRecords {323            bytes += be(Int32(t * 6))324            for la in 0..<2 { for lo in 0..<3 { bytes += be(classicTemp(t, la, lo)) } }325            bytes += be([Float(1000.5), 999.25, 998.0][t])326        }327        try? Data(bytes).write(to: url)328        return url329    }330}331
Files main
Menu

Theme