SPB Git

Repository Public License

RobustnessTests.swift 13.8 KB 229 lines swift

1// ORIX Convert — Packages/EngineScientific/Tests/EngineScientificTests/RobustnessTests.swift2// Description: Untrusted input: malformed and truncated NetCDF classic files (own parser), and corrupted or truncated3//              copies of HDF5 fixtures pushed through the whole reading surface (tree, attributes, values) inside4//              the test process, which must end in errors, never in a crash or an unbounded allocation.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation7import Testing8import CHDF59import CHDF5Shim10import ConvertCore11@testable import EngineScientific1213/// Deterministic generator, so a failing mutation can be reproduced from its index.14struct SplitMix64: RandomNumberGenerator {15    var state: UInt6416    mutating func next() -> UInt64 {17        state &+= 0x9E37_79B9_7F4A_7C1518        var z = state19        z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B920        z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB21        return z ^ (z >> 31)22    }23}2425/// Reads everything the engine reaches before converting (the same routine the isolation helper runs).26/// Returns `true` when every object was readable.27@discardableResult28func exercise(_ url: URL) -> Bool { SciProbe.exercise(url).hasSuffix(", 0 unreadable") }2930/// The helper built next to the test bundle by `swift build` / `swift test`.31func probeHelper() -> URL? {32    var candidates = Bundle.allBundles.map { $0.bundleURL.deletingLastPathComponent().appendingPathComponent(SciProbe.toolName) }33    var dir = URL(fileURLWithPath: #filePath)34    for _ in 0..<3 { dir = dir.deletingLastPathComponent() }35    candidates.append(dir.appendingPathComponent(".build/out/Products/Debug/\(SciProbe.toolName)"))36    candidates.append(dir.appendingPathComponent(".build/debug/\(SciProbe.toolName)"))37    return candidates.first { FileManager.default.isExecutableFile(atPath: $0.path) }38}3940@Suite struct ClassicRobustnessTests {41    @Test func malformedHeaders() throws {42        let temp = TempDir()43        let good = [UInt8](try Data(contentsOf: Fixture.classic(temp.file("good.nc"))))44        func fails(_ bytes: [UInt8], _ comment: Comment) {45            let url = temp.file("bad-\(UUID().uuidString.prefix(6)).nc")46            try? Data(bytes).write(to: url)47            #expect(throws: ConversionError.self, comment) { let f = try SciFile(url: url); for n in try f.children() { _ = try f.preview(n.path) } }48        }49        fails(Array("CDF".utf8), "three bytes")50        fails([0x43, 0x44, 0x46, 1], "magic only")51        fails([0x43, 0x44, 0x46, 1, 0, 0, 0, 0], "no lists")52        // Counts that claim more than the file holds.53        var b = good54        b.replaceSubrange(12..<16, with: [0x7F, 0xFF, 0xFF, 0xFF])55        fails(b, "2^31 dimensions")56        b = good57        b.replaceSubrange(16..<20, with: [0x7F, 0xFF, 0xFF, 0xFF])58        fails(b, "a 2 GiB dimension name")59        b = good60        b.replaceSubrange(8..<12, with: [0, 0, 0, 0x55])61        fails(b, "wrong dimension tag")62        // Record count far beyond the file.63        b = good64        b.replaceSubrange(4..<8, with: [0x7F, 0xFF, 0xFF, 0xFF])65        fails(b, "2^31 records")66        // A second record dimension, an unknown type, a dimension id out of range.67        b = good68        b.replaceSubrange(36..<40, with: [0, 0, 0, 0])69        fails(b, "two record dimensions")70        // Truncations at every length: an error, or (when only trailing data is missing) a precise one.71        var failures = 072        for cut in stride(from: 0, to: good.count, by: 3) {73            let url = temp.file("cut.nc")74            try Data(good.prefix(cut)).write(to: url)75            if !exercise(url) { failures += 1 }76        }77        #expect(failures == (good.count + 2) / 3, "every truncation must be detected")78        // The exact message for a file cut in its record data.79        try Data(good.prefix(good.count - 20)).write(to: temp.file("short.nc"))80        #expect(throws: ConversionError.malformedInput("The NetCDF file declares 3 records but holds only 2: it is truncated.")) { _ = try SciFile(url: temp.file("short.nc")) }81        try Data(good.prefix(good.count - 61)).write(to: temp.file("fixed.nc"))82        #expect(throws: ConversionError.malformedInput("The data of the variable time starts outside the file.")) { _ = try SciFile(url: temp.file("fixed.nc")) }83    }8485    @Test func byteFlipsNeverCrash() throws {86        let temp = TempDir()87        var generator = SplitMix64(state: 20261121)88        var opened = 0, refused = 089        for version in [UInt8(1), 2, 5] {90            let good = [UInt8](try Data(contentsOf: Fixture.classic(temp.file("good\(version).nc"), version: version)))91            // Every header byte set to 0x00, 0xFF and 0x7F, one at a time.92            let headerEnd = good.count - 60 - (version == 5 ? 20 : 0)93            for offset in 0..<headerEnd {94                for value in [UInt8(0x00), 0xFF, 0x7F] where good[offset] != value {95                    var b = good96                    b[offset] = value97                    let url = temp.file("flip.nc")98                    try Data(b).write(to: url)99                    if exercise(url) { opened += 1 } else { refused += 1 }100                }101            }102            for _ in 0..<400 {103                var b = good104                for _ in 0..<Int.random(in: 1...6, using: &generator) { b[Int.random(in: 0..<b.count, using: &generator)] = UInt8.random(in: 0...255, using: &generator) }105                let url = temp.file("random.nc")106                try Data(b.prefix(Int.random(in: 0...b.count, using: &generator))).write(to: url)107                if exercise(url) { opened += 1 } else { refused += 1 }108            }109        }110        #expect(refused > 2000 && opened > 100, "\(refused) refused, \(opened) still readable")111    }112113    @Test func classicWriterRefusesWhatDoesNotFit() throws {114        #expect(NCClassicWriter.minimumVersion(dimensions: [NCDimension(name: "n", length: 10)], variables: [NCVariable(name: "v", dimensionIDs: [0], attributes: [], type: .int)], globalAttributes: [], recordCount: 0) == 1)115        #expect(NCClassicWriter.minimumVersion(dimensions: [NCDimension(name: "n", length: 600_000_000)], variables: [NCVariable(name: "v", dimensionIDs: [0], attributes: [], type: .int)], globalAttributes: [], recordCount: 0) == 2)116        #expect(NCClassicWriter.minimumVersion(dimensions: [NCDimension(name: "n", length: 600_000_000)], variables: [NCVariable(name: "v", dimensionIDs: [0], attributes: [], type: .double)], globalAttributes: [], recordCount: 0) == 5)117        #expect(NCClassicWriter.minimumVersion(dimensions: [], variables: [NCVariable(name: "v", dimensionIDs: [], attributes: [], type: .uint64)], globalAttributes: [], recordCount: 0) == 5)118        #expect(NCClassicWriter.minimumVersion(dimensions: [], variables: [], globalAttributes: [NCAttribute(name: "a", type: .ubyte, data: Data([1]))], recordCount: 0) == 5)119        // Values that do not fit a type are never wrapped.120        #expect(NCCodec.encode(.int([300]), as: .byte) == nil && NCCodec.encode(.int([-1]), as: .ubyte) == nil && NCCodec.encode(.uint([UInt64.max]), as: .int64) == nil)121        #expect(NCCodec.encode(.int([-128, 127]), as: .byte) == Data([0x80, 0x7F]) && NCCodec.encode(.double([1.5]), as: .int) == nil)122        var used = Set<String>()123        #expect(SciTranscoder.sanitizedName("a/b", used: &used) == "a_b" && SciTranscoder.sanitizedName("a_b", used: &used) == "a_b_2" && SciTranscoder.sanitizedName("-x ", used: &used) == "_-x")124        #expect(NCClassicWriter.validName("temp_2m") && !NCClassicWriter.validName("a/b") && !NCClassicWriter.validName("") && !NCClassicWriter.validName("trailing "))125    }126}127128@Suite(.serialized) struct HDF5RobustnessTests {129    @Test func notHDF5() throws {130        let temp = TempDir()131        for (name, data) in [("empty.h5", Data()), ("text.h5", Data("this is not HDF5".utf8)), ("zeros.h5", Data(count: 4096)), ("signature.h5", Data([0x89, 0x48, 0x44, 0x46, 0x0D, 0x0A, 0x1A, 0x0A]))] {132            let url = temp.file(name)133            try data.write(to: url)134            #expect(throws: ConversionError.self) { _ = try SciFile(url: url) }135        }136        #expect(throws: ConversionError.unreadableInput("missing.h5 cannot be opened.")) { _ = try SciFile(url: temp.file("missing.h5")) }137    }138139    // Damaged HDF5 files are never parsed in the test process: that is what the isolation helper is for. Each file140    // goes to orix-hdf5-probe (its own process, with a deadline), which runs the engine's full reading routine and141    // reports how much was readable.142143    @Test func truncatedFiles() async throws {144        let helper = try #require(probeHelper(), "orix-hdf5-probe was not built next to the tests")145        let temp = TempDir()146        let good = try Data(contentsOf: Fixture.everything(in: temp.url))147        var clean = 0, refused = 0, died = 0148        for cut in stride(from: 0, to: good.count, by: 197) {149            let url = temp.file("cut-\(cut).h5")150            try good.prefix(cut).write(to: url)151            let (outcome, summary) = await SciProbe.runReporting(helper: helper, file: url, timeout: 6)152            if outcome != .passed { died += 1 } else if summary.hasSuffix(", 0 unreadable") { clean += 1 } else { refused += 1 }153            try? FileManager.default.removeItem(at: url)154        }155        print("HDF5 truncation sweep: \(refused) refused with errors, \(died) crashed or hung the helper, \(clean) read as complete")156        #expect(clean == 0, "a truncated file must never read as complete")157        #expect(refused > 100)158    }159160    @Test func corruptedCopiesAreContained() async throws {161        // Random byte damage on two fixtures; mutation indices are deterministic. The C library loops forever on at162        // least one of these files (in H5HG__cache_heap_deserialize, reached through the dimension-scale attributes):163        // that stays inside the helper, which is stopped at its deadline, and this process goes on.164        let helper = try #require(probeHelper(), "orix-hdf5-probe was not built next to the tests")165        let temp = TempDir()166        var generator = SplitMix64(state: 0x0D1F_F00D)167        let fixtures = [try Data(contentsOf: Fixture.everything(in: temp.url)), try Data(contentsOf: Fixture.netCDF4(temp.file("raw.nc")))]168        var readable = 0, damaged = 0, crashed = 0, hung = 0169        for (f, good) in fixtures.enumerated() {170            for i in 0..<300 {171                var b = good172                let flips = 1 + i % 8173                for _ in 0..<flips {174                    // Half of the damage goes to the first 4 KiB, where the superblock and root structures live.175                    let limit = i % 2 == 0 ? min(4096, b.count) : b.count176                    b[Int.random(in: 0..<limit, using: &generator)] = UInt8.random(in: 0...255, using: &generator)177                }178                let url = temp.file("mutant-\(f)-\(i).h5")179                try b.write(to: url)180                let (outcome, summary) = await SciProbe.runReporting(helper: helper, file: url, timeout: 6)181                switch outcome {182                case .crashed: crashed += 1183                case .timedOut: hung += 1184                case .passed: if summary.hasSuffix(", 0 unreadable") { readable += 1 } else { damaged += 1 }185                }186                try? FileManager.default.removeItem(at: url)187            }188        }189        print("HDF5 corruption sweep: \(readable) fully readable, \(damaged) partly or wholly refused with errors, \(crashed) crashed the helper, \(hung) hung the helper")190        #expect(readable + damaged + crashed + hung == 600 && damaged > 400)191        // The library in this process is untouched and usable.192        let file = try SciFile(url: Fixture.single(temp.file("after.h5")))193        #expect(try file.preview("/m").values.count == 12)194    }195196    @Test func stalledAndCrashedHelpersAreReported() async throws {197        // What the engine relies on: a helper that stalls is reported as timed out, one that dies as crashed, and198        // only a helper that wrote its status file counts as passed.199        let helper = try #require(probeHelper())200        let temp = TempDir()201        // Stand-in helpers that never finish, or die on a signal, show the contract without depending on a specific202        // defect of the library.203        let tools = temp.sub("tools")204        let stall = tools.appendingPathComponent(SciProbe.toolName)205        try Data("#!/bin/sh\nsleep 30\n".utf8).write(to: stall)206        try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: stall.path)207        let source = Fixture.single(temp.file("s.h5"))208        #expect(await SciProbe.run(helper: stall, file: source, timeout: 1) == .timedOut)209        let crash = tools.appendingPathComponent("crash")210        try Data("#!/bin/sh\nkill -SEGV $$\n".utf8).write(to: crash)211        try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: crash.path)212        if case .crashed = await SciProbe.run(helper: crash, file: source, timeout: 5) {} else { Issue.record("a helper killed by a signal must be reported as crashed") }213        #expect(await SciProbe.run(helper: helper, file: source, timeout: 20) == .passed)214    }215216    @Test func helperContract() async throws {217        let helper = try #require(probeHelper())218        let temp = TempDir()219        #expect(await SciProbe.run(helper: helper, file: Fixture.everything(in: temp.url)) == .passed)220        // A file that is not HDF5 at all is a clean error, not a crash.221        try Data("nothing".utf8).write(to: temp.file("text.h5"))222        #expect(await SciProbe.run(helper: helper, file: temp.file("text.h5")) == .passed)223        // Classic files never need the helper; HDF5 files do.224        #expect(!SciProbe.needsProbe(Fixture.classic(temp.file("c.nc"))) && SciProbe.needsProbe(temp.file("everything.h5")))225        // Without the helper in the bundle the engine says so in its log and reads in-process.226        #expect(SciProbe.isAvailable == (BundledTools.url(for: SciProbe.toolName) != nil))227    }228}229
Files main
Menu

Theme