SPB Git

Repository Public License

RobustnessTests.swift 15.9 KB 230 lines swift

1// ORIX Convert — Packages/EngineGeo/Tests/EngineGeoTests/RobustnessTests.swift2// Description: Untrusted input: malformed and truncated files for every reader, XML entity tricks, lying length3//              fields, and byte-level corruption sweeps that must end in an error, never in a crash.4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Foundation6import Testing7import ConvertCore8@testable import EngineGeo910@Suite struct RobustnessTests {11    static func fails(_ url: URL, _ format: FormatID, _ comment: Comment) {12        #expect(throws: ConversionError.self, comment) { _ = try GeoReader.read(url, format: format, options: GeoReadOptions(), check: {}) }13    }1415    @Test func malformedGeoJSON() {16        let temp = TempDir()17        let cases = ["", "{", "[1,2,3]", "{\"type\":\"Nope\"}", "{\"type\":\"FeatureCollection\"}", "{\"type\":\"Feature\",\"geometry\":{\"type\":\"Point\"}}",18                     "{\"type\":\"Point\",\"coordinates\":[1]}", "{\"type\":\"Point\",\"coordinates\":[\"a\",\"b\"]}", "{\"type\":\"Polygon\",\"coordinates\":[[1,2]]}",19                     "{\"type\":\"Point\",\"coordinates\":[1,2]} trailing", "{\"type\":\"Point\",\"coordinates\":[1,2],}", String(repeating: "[", count: 5000)]20        for (i, text) in cases.enumerated() { Self.fails(temp.write("bad\(i).geojson", text), .geojson, "case \(i)") }21        // Deep nesting of collections is bounded.22        let deep = String(repeating: "{\"type\":\"GeometryCollection\",\"geometries\":[", count: 40) + String(repeating: "]}", count: 40)23        Self.fails(temp.write("deep.geojson", deep), .geojson, "deep collections")24    }2526    @Test func malformedXML() {27        let temp = TempDir()28        Self.fails(temp.write("a.kml", "<kml><Document><Placemark></kml>"), .kml, "unbalanced")29        Self.fails(temp.write("b.kml", "<html><body/></html>"), .kml, "wrong root")30        Self.fails(temp.write("c.kml", "<kml><Placemark><Point><coordinates>abc,def</coordinates></Point></Placemark></kml>"), .kml, "bad coordinates")31        Self.fails(temp.write("d.gpx", "<gpx><wpt lat=\"x\" lon=\"1\"/></gpx>"), .gpx, "bad latitude")32        Self.fails(temp.write("e.gpx", "<kml/>"), .gpx, "wrong root")33        Self.fails(temp.write("f.gpx", ""), .gpx, "empty")34        // Entity expansion and external entities are refused outright.35        let bomb = "<?xml version=\"1.0\"?><!DOCTYPE kml [<!ENTITY a \"aaaaaaaaaa\"><!ENTITY b \"&a;&a;&a;&a;&a;&a;&a;&a;\"><!ENTITY c \"&b;&b;&b;&b;&b;&b;&b;&b;\">]><kml><Placemark><name>&c;</name></Placemark></kml>"36        #expect(throws: ConversionError.malformedInput("The XML document declares its own entities, which is not accepted.")) { _ = try read(temp.write("bomb.kml", bomb), .kml) }37        let secret = temp.write("secret.txt", "TOP-SECRET")38        let xxe = "<?xml version=\"1.0\"?><!DOCTYPE gpx [<!ENTITY x SYSTEM \"file://\(secret.path)\">]><gpx><wpt lat=\"1\" lon=\"2\"><name>&x;</name></wpt></gpx>"39        do {40            let d = try read(temp.write("xxe.gpx", xxe), .gpx)41            #expect(!d.layers.flatMap(\.features).contains { $0["name"] == .string("TOP-SECRET") }, "an external entity was resolved")42        } catch {}43        // Nesting deeper than the bound.44        let deep = "<kml>" + String(repeating: "<Folder>", count: 300) + String(repeating: "</Folder>", count: 300) + "</kml>"45        #expect(throws: ConversionError.limitExceeded("The XML document is nested deeper than 128 levels.")) { _ = try read(temp.write("deep.kml", deep), .kml) }46    }4748    @Test func malformedCSV() {49        let temp = TempDir()50        Self.fails(temp.write("a.csv", ""), .csv, "empty")51        Self.fails(temp.write("b.csv", "lat,lon\n\"45,-73\n"), .csv, "unterminated quote")52        Self.fails(temp.write("c.csv", "lat,lon\nnorth,west\n"), .csv, "text coordinates")53        Self.fails(temp.write("d.csv", "id,wkt\n1,POINT (1)\n"), .csv, "bad WKT")54        Self.fails(temp.write("e.csv", "id,wkt\n1,\"POLYGON ((0 0, 1 1\"\n"), .csv, "unterminated WKT")55    }5657    @Test func wktAndWKBParsersRejectGarbage() throws {58        for text in ["POINT", "POINT (1 2", "POINT (1 2 3 4 5)", "LINESTRING (1 2, x y)", "CIRCULARSTRING (0 0, 1 1, 2 0)", "POINT (1 2) junk", String(repeating: "GEOMETRYCOLLECTION (", count: 40)] {59            #expect(throws: ConversionError.self, "\(text.prefix(30))") { _ = try GeometryWKT.parse(text) }60        }61        #expect(try GeometryWKT.parse("SRID=4326;POINT(1 2)") == .point(Coordinate(1, 2)))62        #expect(try GeometryWKT.parse("point empty") == nil)63        #expect(try GeometryWKT.parse("MULTIPOINT (1 2, 3 4)") == .multiPoint([Coordinate(1, 2), Coordinate(3, 4)]))64        #expect(try GeometryWKT.parse("POINT M (1 2 3)") == .point(Coordinate(1, 2, m: 3)))65        // Round trip of every geometry type through WKT and WKB.66        let all: [Geometry] = [67            .point(Coordinate(1.5, -2.25, z: 3, m: 4)), .lineString([Coordinate(0, 0), Coordinate(1e-7, 123_456_789.125)]),68            .polygon([[Coordinate(0, 0), Coordinate(1, 0), Coordinate(1, 1), Coordinate(0, 0)], [Coordinate(0.2, 0.1), Coordinate(0.8, 0.1), Coordinate(0.8, 0.7), Coordinate(0.2, 0.1)]]),69            .multiPoint([Coordinate(1, 2, z: 3), Coordinate(4, 5, z: 6)]), .multiLineString([[Coordinate(0, 0), Coordinate(1, 1)], [Coordinate(2, 2), Coordinate(3, 3)]]),70            .multiPolygon([[[Coordinate(0, 0), Coordinate(1, 0), Coordinate(1, 1), Coordinate(0, 0)]], [[Coordinate(5, 5), Coordinate(6, 5), Coordinate(6, 6), Coordinate(5, 5)]]]),71            .collection([.point(Coordinate(1, 2)), .collection([.lineString([Coordinate(0, 0), Coordinate(1, 1)])])]),72        ]73        for g in all {74            #expect(try GeometryWKT.parse(GeometryWKT.write(g)) == g, "WKT \(g.typeName)")75            #expect(try GeometryWKB.read(GeometryWKB.write(g)) == g, "WKB \(g.typeName)")76        }77        // WKB: a count larger than the remaining bytes is refused before anything is allocated.78        let lying = Data([1, 2, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0x7F] + [UInt8](repeating: 0, count: 32))79        #expect(throws: ConversionError.self) { _ = try GeometryWKB.read(lying) }80        #expect(throws: ConversionError.self) { _ = try GeometryWKB.read(Data([1, 99, 0, 0, 0])) }81        #expect(throws: ConversionError.self) { _ = try GeometryWKB.read(Data([7])) }82        // EWKB with SRID and Z flags (PostGIS style) is understood.83        let ewkb = Data([1, 1, 0, 0, 0xA0, 0xE6, 0x10, 0, 0] + RawGeoPackage.le(1) + RawGeoPackage.le(2) + RawGeoPackage.le(3))84        #expect(try GeometryWKB.read(ewkb) == .point(Coordinate(1, 2, z: 3)))85    }8687    @Test func malformedShapefiles() throws {88        let temp = TempDir()89        let dir = temp.sub("bad")90        func shp(_ name: String, _ data: Data) -> URL { let u = dir.appendingPathComponent(name); try? data.write(to: u); return u }91        Self.fails(shp("empty.shp", Data()), .shapefile, "empty")92        Self.fails(shp("short.shp", Data(count: 60)), .shapefile, "short")93        Self.fails(shp("magic.shp", Data(count: 200)), .shapefile, "no magic")94        let good = ShapefileBytes.shp(type: 5, shapes: [.poly(type: 5, parts: [ShapefileTests.outerA, ShapefileTests.holeA])])95        // Truncated in the middle of the record.96        Self.fails(shp("cut.shp", good.prefix(good.count - 20)), .shapefile, "truncated")97        // numPoints patched to 2^31 − 1 and numParts to a huge value.98        var lying = good99        lying.replaceSubrange(148..<152, with: [0xFF, 0xFF, 0xFF, 0x7F])100        Self.fails(shp("points.shp", lying), .shapefile, "lying point count")101        lying = good102        lying.replaceSubrange(144..<148, with: [0xFF, 0xFF, 0xFF, 0x7F])103        Self.fails(shp("parts.shp", lying), .shapefile, "lying part count")104        // Negative counts and part offsets out of order.105        lying = good106        lying.replaceSubrange(148..<152, with: [0xFF, 0xFF, 0xFF, 0xFF])107        Self.fails(shp("negative.shp", lying), .shapefile, "negative count")108        lying = good109        lying.replaceSubrange(156..<160, with: [0x64, 0, 0, 0])110        Self.fails(shp("offsets.shp", lying), .shapefile, "part offset past the points")111        // Record length larger than the file.112        lying = good113        lying.replaceSubrange(104..<108, with: [0x7F, 0xFF, 0xFF, 0xFF])114        Self.fails(shp("length.shp", lying), .shapefile, "lying record length")115        // MultiPatch is named as unsupported.116        var patch = good117        patch.replaceSubrange(32..<36, with: [31, 0, 0, 0])118        #expect(throws: ConversionError.malformedInput("patch.shp holds MultiPatch (3D surface) shapes, which are not supported.")) { _ = try read(shp("patch.shp", patch), .shapefile) }119        // A damaged dBASE table next to a good .shp.120        _ = shp("table.shp", ShapefileBytes.shp(type: 1, shapes: [.point(x: 1, y: 2)]))121        try Data([3, 1, 1, 1, 0xFF, 0xFF, 0xFF, 0x7F, 0x41, 0, 0x05, 0] + [UInt8](repeating: 0, count: 60)).write(to: dir.appendingPathComponent("table.dbf"))122        let d = try read(dir.appendingPathComponent("table.shp"), .shapefile)123        #expect(d.layers[0].features.count == 1)124        #expect(d.warnings.contains { $0.code == "dbf-truncated" })125        try Data([3, 1, 1]).write(to: dir.appendingPathComponent("table.dbf"))126        Self.fails(dir.appendingPathComponent("table.shp"), .shapefile, "dbf too short")127    }128129    @Test func malformedGeoPackages() throws {130        let temp = TempDir()131        Self.fails(temp.write("text.gpkg", "this is not a database"), .geopackage, "not SQLite")132        // SQLite but not a GeoPackage.133        let plain = temp.file("plain.gpkg")134        RawGeoPackage.create(plain, sql: "DROP TABLE gpkg_contents;")135        #expect(throws: ConversionError.malformedInput("The SQLite database has no gpkg_contents table: it is not a GeoPackage.")) { _ = try read(plain, .geopackage) }136        // gpkg_contents pointing at a missing table, and a feature table missing from gpkg_geometry_columns.137        let ghost = temp.file("ghost.gpkg")138        RawGeoPackage.create(ghost, sql: "INSERT INTO gpkg_contents (table_name, data_type, identifier, srs_id) VALUES ('ghost', 'features', 'ghost', 4326);")139        Self.fails(ghost, .geopackage, "missing table")140        // Geometry blobs that are not GeoPackageBinary, or that lie about their contents.141        let blobs: [[UInt8]] = [Array("garbage".utf8), [0x47, 0x50, 0, 0x0B, 0, 0, 0, 0], Array(RawGeoPackage.header(srs: 4326, envelope: 1).dropLast(4)),142                                RawGeoPackage.header(srs: 4326) + [1, 2, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF]]143        for (i, blob) in blobs.enumerated() {144            let url = temp.file("blob\(i).gpkg")145            RawGeoPackage.create(url, sql: """146            CREATE TABLE t (fid INTEGER PRIMARY KEY, geom BLOB);147            INSERT INTO gpkg_contents (table_name, data_type, identifier, srs_id) VALUES ('t', 'features', 't', 4326);148            INSERT INTO gpkg_geometry_columns VALUES ('t', 'geom', 'GEOMETRY', 4326, 0, 0);149            """, inserts: [("INSERT INTO t (geom) VALUES (?)", blob)])150            Self.fails(url, .geopackage, "blob \(i)")151        }152        // A hostile table name is only ever used as a quoted identifier.153        let hostile = temp.file("hostile.gpkg")154        RawGeoPackage.create(hostile, sql: """155        CREATE TABLE "x""; DROP TABLE gpkg_contents; --" (fid INTEGER PRIMARY KEY, geom BLOB, v TEXT);156        INSERT INTO "x""; DROP TABLE gpkg_contents; --" (v) VALUES ('ok');157        INSERT INTO gpkg_contents (table_name, data_type, identifier, srs_id) VALUES ('x"; DROP TABLE gpkg_contents; --', 'features', 'h', 4326);158        INSERT INTO gpkg_geometry_columns VALUES ('x"; DROP TABLE gpkg_contents; --', 'geom', 'GEOMETRY', 4326, 0, 0);159        """)160        let d = try read(hostile, .geopackage)161        #expect(d.layers[0].features[0]["v"] == .string("ok"))162        #expect(try read(hostile, .geopackage).layers.count == 1, "the contents table is still there")163    }164165    @Test func corruptionSweepNeverCrashes() throws {166        // Flip bytes across each binary fixture; the reader must either succeed or throw.167        let temp = TempDir()168        let dir = temp.sub("sweep")169        let shpURL = Cities.writeShapefile(in: dir)170        let shpData = try Data(contentsOf: shpURL), dbfData = try Data(contentsOf: dir.appendingPathComponent("cities.dbf"))171        let polygon = ShapefileBytes.shp(type: 15, shapes: [.polyZ(type: 15, parts: [ShapefileTests.outerA, ShapefileTests.holeA], z: Array(repeating: 1, count: 10), m: Array(repeating: 2, count: 10))])172        var attempts = 0, failures = 0173        var generator = SystemRandomNumberGenerator()174        func attempt(_ body: () throws -> Void) { attempts += 1; do { try body() } catch { failures += 1 } }175        for offset in stride(from: 0, to: polygon.count, by: 1) {176            for value in [UInt8(0x00), 0xFF, 0x7F, 0x80] {177                var d = polygon178                d[offset] = value179                attempt { _ = try ShapefileFormat.read(ShapefileParts(name: "p", shp: d, dbf: nil, prj: nil, cpg: nil), encodingOverride: "", check: {}) }180            }181        }182        for _ in 0..<600 {183            var s = shpData, t = dbfData184            for _ in 0..<3 { s[Int.random(in: 0..<s.count, using: &generator)] = UInt8.random(in: 0...255, using: &generator) }185            for _ in 0..<3 { t[Int.random(in: 0..<t.count, using: &generator)] = UInt8.random(in: 0...255, using: &generator) }186            attempt { _ = try ShapefileFormat.read(ShapefileParts(name: "c", shp: s, dbf: t, prj: nil, cpg: nil), encodingOverride: "", check: {}) }187            attempt { _ = try ShapefileFormat.read(ShapefileParts(name: "c", shp: s.prefix(Int.random(in: 0...s.count, using: &generator)), dbf: t.prefix(Int.random(in: 0...t.count, using: &generator)), prj: nil, cpg: nil), encodingOverride: "", check: {}) }188        }189        // WKB and GeoPackage binary.190        let wkb = GeometryWKB.write(.multiPolygon([[ShapefileTests.ring(ShapefileTests.outerA), ShapefileTests.ring(ShapefileTests.holeA)], [ShapefileTests.ring(ShapefileTests.outerB)]]))191        for offset in 0..<wkb.count {192            for value in [UInt8(0x00), 0xFF, 0x7F] {193                var d = wkb194                d[offset] = value195                attempt { _ = try GeometryWKB.read(d) }196                attempt { _ = try GeoPackageFormat.decodeGeometry(Data(RawGeoPackage.header(srs: 1)) + d.prefix(offset)) }197            }198        }199        // ZIP containers.200        let zip = ZipBytes.archive([("doc.kml", Data(Samples.kml.utf8), true), ("a.txt", Data("x".utf8), false)])201        for _ in 0..<400 {202            var d = zip203            for _ in 0..<2 { d[Int.random(in: 0..<d.count, using: &generator)] = UInt8.random(in: 0...255, using: &generator) }204            attempt { let z = try ZipArchive(d); for e in z.entries { _ = try z.extract(e) } }205        }206        // Text formats cut at every length.207        for text in [Cities.geojson, Cities.kml, Samples.gpx, Cities.csv] {208            let bytes = Array(text.utf8)209            for cut in stride(from: 0, to: bytes.count, by: 7) {210                let d = Data(bytes.prefix(cut))211                attempt { _ = try GeoJSONFormat.read([UInt8](d), layerName: "x", check: {}) }212                attempt { _ = try KMLFormat.read(d, defaultName: "x", check: {}) }213                attempt { _ = try GPXFormat.read(d, trackPointsAsFeatures: true, check: {}) }214                attempt { _ = try CSVFormat.read(d, layerName: "x", options: CSVReadOptions(), check: {}) }215            }216        }217        #expect(attempts > 3000 && failures > 500, "\(attempts) attempts, \(failures) clean failures")218    }219220    @Test func unreadableAndOversizedInputs() async {221        let temp = TempDir()222        Self.fails(temp.file("missing.geojson"), .geojson, "missing file")223        Self.fails(temp.file("missing.gpkg"), .geopackage, "missing database")224        Self.fails(temp.file("missing.shp"), .shapefile, "missing shapefile")225        await #expect(throws: ConversionError.self) {226            _ = try await GeoEngine().convert(ConversionRequest(source: temp.file("x.png"), sourceFormat: .png, destinationFormat: .geojson), context: ConversionContext(workDirectory: temp.url))227        }228    }229}230
Files main
Menu

Theme