SPB Git

Repository Public License

ArchiveTests.swift 20.0 KB 385 lines swift

1// ORIX Secure2// File: Packages/ORIXFileInspection/Tests/ORIXFileInspectionTests/ArchiveTests.swift3// Description: ZIP central-directory inspection on hand-built archives (declared sizes, traversal, ZIP64, lies and4//              damage), the TAR header walk and the GZIP header.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import ORIXSecureModel10import Testing11@testable import ORIXFileInspection1213@Suite("Archives without extraction")14struct ArchiveTests {15    func findings(_ data: [UInt8], name: String = "archive.zip", options: InspectionOptions = .init()) throws -> (FileInspector.CoreInspection, ArchiveReport) {16        let core = try inspectBytes(data, name: name, options: options)17        expectHonest(core.findings)18        let report = try #require(core.archive)19        return (core, report)20    }2122    @Test("A plain stored archive is described exactly")23    func plainArchive() throws {24        let (core, report) = try findings(Samples.plainZip)25        #expect(report.format == .zip)26        #expect(report.isWellFormed)27        #expect(report.declaredEntryCount == 3)28        #expect(report.parsedEntryCount == 3)29        #expect(report.directoryCount == 1)30        #expect(report.totalCompressedSize == 13)31        #expect(report.totalUncompressedSize == 13)32        #expect(report.overallCompressionRatio == 1)33        #expect(report.compressionMethods == ["stored": 3])34        #expect(report.entries.map(\.path) == ["readme.txt", "folder/", "folder/data.csv"])35        #expect(report.notableEntries.isEmpty)36        #expect(!core.findings.contains { $0.state == .attention })37    }3839    @Test("An empty archive is well formed")40    func emptyArchive() throws {41        let (core, report) = try findings(buildZip([]))42        #expect(report.isWellFormed)43        #expect(report.parsedEntryCount == 0)44        #expect(report.overallCompressionRatio == nil)45        #expect(!core.findings.contains { $0.state == .attention })46    }4748    @Test("Decompression bomb: declared sizes produce findings and nothing is inflated")49    func bomb() throws {50        // 12 entries of 16 stored bytes, each declaring 4 GB of deflated content.51        let entries = (0..<12).map { ZipEntrySpec("layer\($0).bin", zeros(16), method: 8, declaredCompressed: 4096, declaredUncompressed: 4_000_000_000) }52        let data = buildZip(entries)53        #expect(data.count < 2000)54        let start = Date()55        let (core, report) = try findings(data, name: "small.zip")56        #expect(Date().timeIntervalSince(start) < 1)57        #expect(report.totalUncompressedSize == 48_000_000_000)58        let entryRatio = try #require(report.highestEntryCompressionRatio)59        #expect(entryRatio > 900_000)60        let entry = try #require(core.findings.with("file.zip.ratio.entry"))61        #expect(entry.severity == .high)62        #expect(entry.confidence == .measured)63        #expect(core.findings.with("file.zip.ratio.overall")?.severity == .moderate)64        #expect(core.findings.has("file.zip.declaredSize"))65        #expect(report.notableEntries.contains { $0.flags.contains(.extremeRatio) })66    }6768    @Test("Ratio thresholds come from the options")69    func ratioThresholds() throws {70        let data = buildZip([ZipEntrySpec("log.txt", zeros(8), method: 8, declaredCompressed: 1000, declaredUncompressed: 150_000)])71        let (core, _) = try findings(data)72        #expect(core.findings.has("file.zip.ratio.overall"))73        #expect(!core.findings.has("file.zip.ratio.entry"))74        var options = InspectionOptions()75        options.maxOverallCompressionRatio = 20076        let (relaxed, _) = try findings(data, options: options)77        #expect(!relaxed.findings.has("file.zip.ratio.overall"))78        options.maxEntryCompressionRatio = 10079        let (strict, _) = try findings(data, options: options)80        #expect(strict.findings.has("file.zip.ratio.entry"))81    }8283    @Test("Overlapping entries (several names for one block of data)")84    func overlap() throws {85        let entries = [ZipEntrySpec("a", zeros(10))] + (1..<6).map { ZipEntrySpec("copy\($0)", zeros(10), sharesDataWith: 0) }86        let (core, report) = try findings(buildZip(entries))87        #expect(report.overlappingEntryCount == 5)88        #expect(core.findings.with("file.zip.overlap")?.severity == .high)89    }9091    @Test("Path traversal: parent references, absolute paths, drive letters")92    func traversal() throws {93        let names = ["../../etc/passwd", "/absolute/path.txt", "C:\\Windows\\evil.txt", "ok/../../up.txt", "..\\..\\win.txt", "fine/a..b/file.txt", "also..fine.txt", "dir/"]94        let (core, report) = try findings(buildZip(names.map { ZipEntrySpec($0, bytes("x")) }))95        #expect(report.pathTraversalCount == 5)96        let finding = try #require(core.findings.with("file.zip.traversal"))97        #expect(finding.severity == .high)98        #expect(finding.evidence.contains { $0.value.contains("../../etc/passwd") })99        #expect(ArchiveNames.isTraversal("a/b/c.txt") == false)100        #expect(ArchiveNames.isTraversal("a/..") == true)101    }102103    @Test("Encrypted flag, symbolic links, duplicate names")104    func flags() throws {105        let entries = [106            ZipEntrySpec("secret.txt", zeros(20), flags: 0x0801),107            ZipEntrySpec("link", bytes("/etc/hosts"), externalAttributes: UInt32(0o120777) << 16),108            ZipEntrySpec("same.txt", bytes("1")),109            ZipEntrySpec("same.txt", bytes("2")),110            ZipEntrySpec("regular.txt", bytes("3"), externalAttributes: UInt32(0o100644) << 16),111        ]112        let (core, report) = try findings(buildZip(entries))113        #expect(report.encryptedEntryCount == 1)114        #expect(report.symbolicLinkCount == 1)115        #expect(report.duplicateNameCount == 1)116        #expect(core.findings.with("file.zip.encrypted")?.severity == .low)117        #expect(core.findings.has("file.zip.symlinks"))118        #expect(core.findings.with("file.zip.duplicates")?.severity == .moderate)119        #expect(core.findings.with("file.zip.encrypted")?.notInspected.isEmpty == false)120    }121122    @Test("Nested archives and programs inside")123    func nestedAndPrograms() throws {124        let entries = [ZipEntrySpec("inner.zip", zeros(4)), ZipEntrySpec("deep/more.7z", zeros(4)), ZipEntrySpec("Invoice.PDF.EXE", zeros(4)), ZipEntrySpec("run.command", zeros(4)), ZipEntrySpec("doc.pdf", zeros(4))]125        let (core, report) = try findings(buildZip(entries))126        #expect(report.nestedArchiveCount == 2)127        #expect(report.executableCount == 2)128        #expect(core.findings.has("file.zip.nested"))129        let programs = try #require(core.findings.with("file.zip.executables"))130        #expect(programs.confidence == .heuristic)131        // Programs are expected inside an app package: no such finding there.132        let ipa = try inspectBytes(buildZip([ZipEntrySpec("Payload/A.app/A", zeros(4)), ZipEntrySpec("Payload/A.app/lib.dylib", zeros(4))]), name: "a.ipa")133        #expect(ipa.type == .ipa)134        #expect(!ipa.findings.has("file.zip.executables"))135    }136137    @Test("ZIP64 record, locator and extra fields")138    func zip64() throws {139        let entries = [140            ZipEntrySpec("huge.bin", zeros(8), method: 8, declaredCompressed: 5_000_000_000, declaredUncompressed: 9_000_000_000, zip64: true),141            ZipEntrySpec("small.txt", bytes("hello")),142        ]143        let (_, report) = try findings(buildZip(entries, zip64Record: true))144        #expect(report.usesZip64)145        #expect(report.isWellFormed)146        #expect(report.declaredEntryCount == 2)147        #expect(report.parsedEntryCount == 2)148        #expect(report.totalUncompressedSize == 9_000_000_005)149        #expect(report.totalCompressedSize == 5_000_000_005)150        // ZIP64 markers without the record.151        var broken = buildZip(entries)152        let eocd = broken.count - 22153        broken.replaceSubrange((eocd + 8)..<(eocd + 12), with: [0xFF, 0xFF, 0xFF, 0xFF])154        broken.replaceSubrange((eocd + 16)..<(eocd + 20), with: [0xFF, 0xFF, 0xFF, 0xFF])155        let (core, damaged) = try findings(broken)156        #expect(!damaged.isWellFormed)157        #expect(core.findings.has("file.zip.malformed"))158        // A locator pointing far outside the file.159        var wild = buildZip(entries, zip64Record: true)160        let locator = wild.count - 22 - 20161        wild.replaceSubrange((locator + 8)..<(locator + 16), with: le64(0xFFFF_FFFF_FFFF_FFF0))162        let (_, wildReport) = try findings(wild)163        #expect(!wildReport.isWellFormed)164    }165166    @Test("Truncated archive and garbage end record become findings, not errors")167    func truncatedAndGarbage() throws {168        let truncated = Array(Samples.plainZip.dropLast(10))169        let (core, report) = try findings(truncated)170        #expect(!report.isWellFormed)171        #expect(report.malformation != nil)172        #expect(core.findings.with("file.zip.malformed")?.confidence == .measured)173174        var garbage = Samples.plainZip175        let eocd = garbage.count - 22176        garbage.replaceSubrange((eocd + 12)..<(eocd + 20), with: le32(0x7FFF_FFFF) + le32(0xFFFF_FF00))177        let (second, secondReport) = try findings(garbage)178        #expect(!secondReport.isWellFormed)179        #expect(second.findings.has("file.zip.malformed"))180181        // A directory that turns to noise half-way.182        var noisy = Samples.plainZip183        let directory = Int(try #require(noisy.le32(eocd + 16)))184        for index in (directory + 56)..<(directory + 66) { noisy[index] = 0xEE }185        let (third, thirdReport) = try findings(noisy)186        #expect(!thirdReport.isWellFormed)187        #expect(thirdReport.parsedEntryCount == 1)188        #expect(third.findings.has("file.zip.malformed"))189        #expect(third.findings.has("file.zip.entryCountMismatch"))190191        // Local header only: the end record is missing altogether.192        let headerOnly = Array(Samples.plainZip.prefix(40))193        let (fourth, _) = try findings(headerOnly)194        #expect(fourth.findings.has("file.zip.malformed"))195    }196197    @Test("The end record lies about the number of entries")198    func entryCountLies() throws {199        let entries = (0..<3).map { ZipEntrySpec("f\($0).txt", bytes("x")) }200        let (more, moreReport) = try findings(buildZip(entries, declaredCount: 9))201        #expect(moreReport.declaredEntryCount == 9)202        #expect(moreReport.parsedEntryCount == 3)203        let finding = try #require(more.findings.with("file.zip.entryCountMismatch"))204        #expect(finding.evidence.contains { $0.value == "9" })205        #expect(finding.evidence.contains { $0.value == "3" })206        let (fewer, fewerReport) = try findings(buildZip(entries, declaredCount: 1))207        #expect(fewerReport.parsedEntryCount == 3)208        #expect(fewer.findings.has("file.zip.entryCountMismatch"))209        let (honest, _) = try findings(buildZip(entries))210        #expect(!honest.findings.has("file.zip.entryCountMismatch"))211    }212213    @Test("The walk stops at the entry limit")214    func entryLimit() throws {215        let entries = (0..<60).map { ZipEntrySpec("f\($0).txt", bytes("x")) }216        var options = InspectionOptions()217        options.maxArchiveEntries = 10218        options.maxReportedArchiveEntries = 4219        let (core, report) = try findings(buildZip(entries), options: options)220        #expect(report.parsedEntryCount == 10)221        #expect(report.entriesTruncated)222        #expect(report.entries.count == 4)223        #expect(core.findings.has("file.zip.tooManyEntries"))224        #expect(!core.findings.has("file.zip.entryCountMismatch"))225    }226227    @Test("A directory of 150 000 entries is cut at 100 000 in bounded time")228    func manyEntries() throws {229        var directory: [UInt8] = []230        directory.reserveCapacity(150_000 * 56)231        for index in 0..<150_000 {232            let name = bytes(String(format: "f%08d", index))233            directory += [0x50, 0x4B, 0x01, 0x02] + le16(0x031E) + le16(20) + le16(0x0800) + le16(8) + le16(0) + le16(0)234            directory += le32(0) + le32(10) + le32(20) + le16(UInt16(name.count)) + le16(0) + le16(0) + le16(0) + le16(0) + le32(0) + le32(UInt32(index))235            directory += name236        }237        let data = directory + [0x50, 0x4B, 0x05, 0x06] + le16(0) + le16(0) + le16(0xFFFE) + le16(0xFFFE) + le32(UInt32(directory.count)) + le32(0) + le16(0)238        let start = Date()239        let inspection = try inspectZip(data)240        let elapsed = Date().timeIntervalSince(start)241        print("ZIP central directory: 100 000 entries of \(data.count / 1_000_000) MB read in \(String(format: "%.2f", elapsed)) s")242        #expect(inspection.report.parsedEntryCount == 100_000)243        #expect(inspection.report.entriesTruncated)244        #expect(inspection.report.entries.count == 500)245        #expect(inspection.report.totalUncompressedSize == 2_000_000)246        #expect(elapsed < 30)247    }248249    @Test("Office documents: macros, required part, embedded and linked parts")250    func office() throws {251        let macro = buildZip([ZipEntrySpec("[Content_Types].xml", bytes("<Types/>")), ZipEntrySpec("word/document.xml", bytes("<d/>")), ZipEntrySpec("word/vbaProject.bin", zeros(64))])252        let hidden = try inspectBytes(macro, name: "contract.docx")253        expectHonest(hidden.findings)254        #expect(hidden.type == .docx)255        #expect(hidden.archive?.containsOfficeMacros == true)256        let finding = try #require(hidden.findings.with("file.ooxml.macrosInMacroFreeExtension"))257        #expect(finding.severity == .high)258        #expect(finding.notInspected.isEmpty == false)259        let declared = try inspectBytes(macro, name: "contract.docm")260        #expect(declared.findings.with("file.ooxml.macros")?.severity == .moderate)261        #expect(!declared.findings.has("file.ooxml.macrosInMacroFreeExtension"))262263        let linked = buildZip([264            ZipEntrySpec("[Content_Types].xml", bytes("<Types/>")), ZipEntrySpec("xl/workbook.xml", bytes("<w/>")),265            ZipEntrySpec("xl/externalLinks/externalLink1.xml", bytes("<e/>")), ZipEntrySpec("xl/embeddings/oleObject1.bin", zeros(8)),266        ])267        let workbook = try inspectBytes(linked, name: "budget.xlsx")268        #expect(workbook.archive?.externalLinkPartCount == 1)269        #expect(workbook.archive?.embeddedObjectPartCount == 1)270        #expect(workbook.findings.has("file.ooxml.external"))271272        let renamed = try inspectBytes(Samples.plainZip, name: "report.xlsx")273        #expect(renamed.findings.has("file.ooxml.contentTypes"))274        let good = try inspectBytes(Samples.docx, name: "letter.docx")275        #expect(!good.findings.contains { $0.state == .attention })276    }277278    @Test("Data placed before the archive is detected")279    func prependedData() throws {280        let stub = Samples.pe + zeros(100)281        let (core, report) = try findings(bytes("PK\u{3}\u{4}") + zeros(996) + Samples.plainZip, name: "sfx.zip")282        #expect(report.prependedByteCount == 1000)283        #expect(report.parsedEntryCount == 3)284        #expect(core.findings.has("file.zip.prepended"))285        // A program followed by an archive is detected as the program.286        let sfx = try inspectBytes(stub + Samples.plainZip, name: "setup.exe")287        #expect(sfx.type == .pe)288    }289290    @Test("Inconsistent sizes and archive comment")291    func sizesAndComment() throws {292        let data = buildZip([ZipEntrySpec("a.bin", zeros(4), declaredCompressed: 0, declaredUncompressed: 500), ZipEntrySpec("b.bin", zeros(4), declaredCompressed: 4, declaredUncompressed: 9)], comment: bytes("made by hand PK\u{5}\u{6} trap"))293        let (core, report) = try findings(data)294        #expect(report.isWellFormed)295        #expect(report.inconsistentSizeCount == 2)296        #expect(core.findings.has("file.zip.sizes"))297    }298299    @Test("Entry names with hidden characters are shown visibly")300    func hostileNames() throws {301        let (_, report) = try findings(buildZip([ZipEntrySpec("evil\u{202E}gpj.exe", zeros(1)), ZipEntrySpec(String(repeating: "A", count: 5000), zeros(1))]))302        #expect(report.entries[0].path == "evil<U+202E>gpj.exe")303        #expect(report.entries[1].path.count == 301)304        // Not valid UTF-8: decoded without failing.305        var raw = buildZip([ZipEntrySpec("abcd.txt", zeros(1))])306        let index = try #require(raw.lastIndex(of: bytes("abcd.txt")))307        raw[index] = 0xE9308        let second = try inspectZip(raw)309        #expect(second.report.entries.first?.path == "ébcd.txt")310    }311312    // MARK: TAR and GZIP313314    @Test("TAR header walk: names, sizes, links, traversal")315    func tar() throws {316        let data = buildTar([317            ("readme.txt", bytes("hello world"), "0"), ("dir/", [], "5"), ("dir/tool.sh", bytes("#!/bin/sh\n"), "0"),318            ("../outside.txt", bytes("x"), "0"), ("link", [], "2"), ("big.bin", [UInt8](repeating: 7, count: 1500), "0"),319        ])320        let (core, report) = try findings(data, name: "bundle.tar")321        #expect(core.type == .tar)322        #expect(report.format == .tar)323        #expect(report.isWellFormed)324        #expect(report.parsedEntryCount == 6)325        #expect(report.totalUncompressedSize == 11 + 10 + 1 + 1500)326        #expect(report.pathTraversalCount == 1)327        #expect(report.symbolicLinkCount == 1)328        #expect(report.executableCount == 1)329        #expect(core.findings.with("file.tar.traversal")?.severity == .high)330        #expect(core.findings.has("file.tar.links"))331    }332333    @Test("TAR: GNU long names, truncation and bad checksums")334    func tarEdgeCases() throws {335        let longName = String(repeating: "d/", count: 80) + "file.txt"336        var data = tarHeader(name: "././@LongLink", size: longName.utf8.count + 1, type: "L")337        data += bytes(longName) + zeros(512 - (longName.utf8.count % 512))338        data += tarHeader(name: "truncated-name", size: 3) + bytes("abc") + zeros(509) + zeros(1024)339        let long = try TarInspector.inspect(source: DataByteSource(data), options: .init())340        #expect(long.parsedEntryCount == 1)341        #expect(long.entries.first?.path == longName)342343        let whole = buildTar([("a.txt", [UInt8](repeating: 1, count: 2000), "0"), ("b.txt", bytes("b"), "0")])344        let cut = try TarInspector.inspect(source: DataByteSource(Array(whole.prefix(1000))), options: .init())345        #expect(!cut.isWellFormed)346        #expect(cut.parsedEntryCount == 1)347        var corrupt = whole348        corrupt[10] = 0x7A349        let (core, bad) = try findings(corrupt, name: "x.tar")350        #expect(!bad.isWellFormed)351        #expect(core.findings.has("file.tar.malformed"))352353        // A size field claiming 8 exabytes must not overflow.354        var huge = tarHeader(name: "huge", size: 0)355        huge.replaceSubrange(124..<136, with: [0x80, 0, 0, 0, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF])356        huge.replaceSubrange(148..<156, with: [UInt8](repeating: 0x20, count: 8))357        let sum = String(huge.reduce(0) { $0 + Int($1) }, radix: 8)358        huge.replaceSubrange(148..<156, with: bytes(String(repeating: "0", count: 6 - sum.count) + sum + "\u{0} "))359        let overflow = try TarInspector.inspect(source: DataByteSource(huge + zeros(1024)), options: .init())360        #expect(!overflow.isWellFormed)361    }362363    @Test("GZIP: original name, declared size and ratio")364    func gzip() throws {365        let header: [UInt8] = [0x1F, 0x8B, 0x08, 0x08, 0, 0, 0, 0, 0, 3] + bytes("installer.exe\u{0}")366        let data = header + zeros(40) + le32(0xDEAD_BEEF) + le32(3_000_000_000)367        let (core, report) = try findings(data, name: "installer.gz")368        #expect(core.type == .gzip)369        #expect(report.originalName == "installer.exe")370        #expect(report.totalUncompressedSize == 3_000_000_000)371        #expect(report.totalCompressedSize == UInt64(data.count))372        let ratio = try #require(core.findings.with("file.gzip.ratio"))373        #expect(ratio.confidence == .heuristic)374        #expect(ratio.severity == .moderate)375        #expect(core.findings.has("file.gzip.originalName"))376377        let modest = [0x1F, 0x8B, 0x08, 0x00, 0, 0, 0, 0, 0, 3] + zeros(100) + le32(0) + le32(400)378        let (plain, plainReport) = try findings(modest, name: "log.gz")379        #expect(plainReport.originalName == nil)380        #expect(!plain.findings.contains { $0.state == .attention })381        let short = try GzipInspector.inspect(source: DataByteSource([0x1F, 0x8B, 0x08]))382        #expect(!short.isWellFormed)383    }384}385
Files main
Menu

Theme