Repository Public License
1// ORIX Secure2// File: Packages/ORIXFileInspection/Tests/ORIXFileInspectionTests/ArchiveTests.swift3// Description: ZIP central-directory inspection on hand-built archives (declared sizes, traversal, ZIP64, lies and4// damage), the TAR header walk and the GZIP header.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import ORIXSecureModel10import Testing11@testable import ORIXFileInspection1213@Suite("Archives without extraction")14struct ArchiveTests {15 func findings(_ data: [UInt8], name: String = "archive.zip", options: InspectionOptions = .init()) throws -> (FileInspector.CoreInspection, ArchiveReport) {16 let core = try inspectBytes(data, name: name, options: options)17 expectHonest(core.findings)18 let report = try #require(core.archive)19 return (core, report)20 }2122 @Test("A plain stored archive is described exactly")23 func plainArchive() throws {24 let (core, report) = try findings(Samples.plainZip)25 #expect(report.format == .zip)26 #expect(report.isWellFormed)27 #expect(report.declaredEntryCount == 3)28 #expect(report.parsedEntryCount == 3)29 #expect(report.directoryCount == 1)30 #expect(report.totalCompressedSize == 13)31 #expect(report.totalUncompressedSize == 13)32 #expect(report.overallCompressionRatio == 1)33 #expect(report.compressionMethods == ["stored": 3])34 #expect(report.entries.map(\.path) == ["readme.txt", "folder/", "folder/data.csv"])35 #expect(report.notableEntries.isEmpty)36 #expect(!core.findings.contains { $0.state == .attention })37 }3839 @Test("An empty archive is well formed")40 func emptyArchive() throws {41 let (core, report) = try findings(buildZip([]))42 #expect(report.isWellFormed)43 #expect(report.parsedEntryCount == 0)44 #expect(report.overallCompressionRatio == nil)45 #expect(!core.findings.contains { $0.state == .attention })46 }4748 @Test("Decompression bomb: declared sizes produce findings and nothing is inflated")49 func bomb() throws {50 // 12 entries of 16 stored bytes, each declaring 4 GB of deflated content.51 let entries = (0..<12).map { ZipEntrySpec("layer\($0).bin", zeros(16), method: 8, declaredCompressed: 4096, declaredUncompressed: 4_000_000_000) }52 let data = buildZip(entries)53 #expect(data.count < 2000)54 let start = Date()55 let (core, report) = try findings(data, name: "small.zip")56 #expect(Date().timeIntervalSince(start) < 1)57 #expect(report.totalUncompressedSize == 48_000_000_000)58 let entryRatio = try #require(report.highestEntryCompressionRatio)59 #expect(entryRatio > 900_000)60 let entry = try #require(core.findings.with("file.zip.ratio.entry"))61 #expect(entry.severity == .high)62 #expect(entry.confidence == .measured)63 #expect(core.findings.with("file.zip.ratio.overall")?.severity == .moderate)64 #expect(core.findings.has("file.zip.declaredSize"))65 #expect(report.notableEntries.contains { $0.flags.contains(.extremeRatio) })66 }6768 @Test("Ratio thresholds come from the options")69 func ratioThresholds() throws {70 let data = buildZip([ZipEntrySpec("log.txt", zeros(8), method: 8, declaredCompressed: 1000, declaredUncompressed: 150_000)])71 let (core, _) = try findings(data)72 #expect(core.findings.has("file.zip.ratio.overall"))73 #expect(!core.findings.has("file.zip.ratio.entry"))74 var options = InspectionOptions()75 options.maxOverallCompressionRatio = 20076 let (relaxed, _) = try findings(data, options: options)77 #expect(!relaxed.findings.has("file.zip.ratio.overall"))78 options.maxEntryCompressionRatio = 10079 let (strict, _) = try findings(data, options: options)80 #expect(strict.findings.has("file.zip.ratio.entry"))81 }8283 @Test("Overlapping entries (several names for one block of data)")84 func overlap() throws {85 let entries = [ZipEntrySpec("a", zeros(10))] + (1..<6).map { ZipEntrySpec("copy\($0)", zeros(10), sharesDataWith: 0) }86 let (core, report) = try findings(buildZip(entries))87 #expect(report.overlappingEntryCount == 5)88 #expect(core.findings.with("file.zip.overlap")?.severity == .high)89 }9091 @Test("Path traversal: parent references, absolute paths, drive letters")92 func traversal() throws {93 let names = ["../../etc/passwd", "/absolute/path.txt", "C:\\Windows\\evil.txt", "ok/../../up.txt", "..\\..\\win.txt", "fine/a..b/file.txt", "also..fine.txt", "dir/"]94 let (core, report) = try findings(buildZip(names.map { ZipEntrySpec($0, bytes("x")) }))95 #expect(report.pathTraversalCount == 5)96 let finding = try #require(core.findings.with("file.zip.traversal"))97 #expect(finding.severity == .high)98 #expect(finding.evidence.contains { $0.value.contains("../../etc/passwd") })99 #expect(ArchiveNames.isTraversal("a/b/c.txt") == false)100 #expect(ArchiveNames.isTraversal("a/..") == true)101 }102103 @Test("Encrypted flag, symbolic links, duplicate names")104 func flags() throws {105 let entries = [106 ZipEntrySpec("secret.txt", zeros(20), flags: 0x0801),107 ZipEntrySpec("link", bytes("/etc/hosts"), externalAttributes: UInt32(0o120777) << 16),108 ZipEntrySpec("same.txt", bytes("1")),109 ZipEntrySpec("same.txt", bytes("2")),110 ZipEntrySpec("regular.txt", bytes("3"), externalAttributes: UInt32(0o100644) << 16),111 ]112 let (core, report) = try findings(buildZip(entries))113 #expect(report.encryptedEntryCount == 1)114 #expect(report.symbolicLinkCount == 1)115 #expect(report.duplicateNameCount == 1)116 #expect(core.findings.with("file.zip.encrypted")?.severity == .low)117 #expect(core.findings.has("file.zip.symlinks"))118 #expect(core.findings.with("file.zip.duplicates")?.severity == .moderate)119 #expect(core.findings.with("file.zip.encrypted")?.notInspected.isEmpty == false)120 }121122 @Test("Nested archives and programs inside")123 func nestedAndPrograms() throws {124 let entries = [ZipEntrySpec("inner.zip", zeros(4)), ZipEntrySpec("deep/more.7z", zeros(4)), ZipEntrySpec("Invoice.PDF.EXE", zeros(4)), ZipEntrySpec("run.command", zeros(4)), ZipEntrySpec("doc.pdf", zeros(4))]125 let (core, report) = try findings(buildZip(entries))126 #expect(report.nestedArchiveCount == 2)127 #expect(report.executableCount == 2)128 #expect(core.findings.has("file.zip.nested"))129 let programs = try #require(core.findings.with("file.zip.executables"))130 #expect(programs.confidence == .heuristic)131 // Programs are expected inside an app package: no such finding there.132 let ipa = try inspectBytes(buildZip([ZipEntrySpec("Payload/A.app/A", zeros(4)), ZipEntrySpec("Payload/A.app/lib.dylib", zeros(4))]), name: "a.ipa")133 #expect(ipa.type == .ipa)134 #expect(!ipa.findings.has("file.zip.executables"))135 }136137 @Test("ZIP64 record, locator and extra fields")138 func zip64() throws {139 let entries = [140 ZipEntrySpec("huge.bin", zeros(8), method: 8, declaredCompressed: 5_000_000_000, declaredUncompressed: 9_000_000_000, zip64: true),141 ZipEntrySpec("small.txt", bytes("hello")),142 ]143 let (_, report) = try findings(buildZip(entries, zip64Record: true))144 #expect(report.usesZip64)145 #expect(report.isWellFormed)146 #expect(report.declaredEntryCount == 2)147 #expect(report.parsedEntryCount == 2)148 #expect(report.totalUncompressedSize == 9_000_000_005)149 #expect(report.totalCompressedSize == 5_000_000_005)150 // ZIP64 markers without the record.151 var broken = buildZip(entries)152 let eocd = broken.count - 22153 broken.replaceSubrange((eocd + 8)..<(eocd + 12), with: [0xFF, 0xFF, 0xFF, 0xFF])154 broken.replaceSubrange((eocd + 16)..<(eocd + 20), with: [0xFF, 0xFF, 0xFF, 0xFF])155 let (core, damaged) = try findings(broken)156 #expect(!damaged.isWellFormed)157 #expect(core.findings.has("file.zip.malformed"))158 // A locator pointing far outside the file.159 var wild = buildZip(entries, zip64Record: true)160 let locator = wild.count - 22 - 20161 wild.replaceSubrange((locator + 8)..<(locator + 16), with: le64(0xFFFF_FFFF_FFFF_FFF0))162 let (_, wildReport) = try findings(wild)163 #expect(!wildReport.isWellFormed)164 }165166 @Test("Truncated archive and garbage end record become findings, not errors")167 func truncatedAndGarbage() throws {168 let truncated = Array(Samples.plainZip.dropLast(10))169 let (core, report) = try findings(truncated)170 #expect(!report.isWellFormed)171 #expect(report.malformation != nil)172 #expect(core.findings.with("file.zip.malformed")?.confidence == .measured)173174 var garbage = Samples.plainZip175 let eocd = garbage.count - 22176 garbage.replaceSubrange((eocd + 12)..<(eocd + 20), with: le32(0x7FFF_FFFF) + le32(0xFFFF_FF00))177 let (second, secondReport) = try findings(garbage)178 #expect(!secondReport.isWellFormed)179 #expect(second.findings.has("file.zip.malformed"))180181 // A directory that turns to noise half-way.182 var noisy = Samples.plainZip183 let directory = Int(try #require(noisy.le32(eocd + 16)))184 for index in (directory + 56)..<(directory + 66) { noisy[index] = 0xEE }185 let (third, thirdReport) = try findings(noisy)186 #expect(!thirdReport.isWellFormed)187 #expect(thirdReport.parsedEntryCount == 1)188 #expect(third.findings.has("file.zip.malformed"))189 #expect(third.findings.has("file.zip.entryCountMismatch"))190191 // Local header only: the end record is missing altogether.192 let headerOnly = Array(Samples.plainZip.prefix(40))193 let (fourth, _) = try findings(headerOnly)194 #expect(fourth.findings.has("file.zip.malformed"))195 }196197 @Test("The end record lies about the number of entries")198 func entryCountLies() throws {199 let entries = (0..<3).map { ZipEntrySpec("f\($0).txt", bytes("x")) }200 let (more, moreReport) = try findings(buildZip(entries, declaredCount: 9))201 #expect(moreReport.declaredEntryCount == 9)202 #expect(moreReport.parsedEntryCount == 3)203 let finding = try #require(more.findings.with("file.zip.entryCountMismatch"))204 #expect(finding.evidence.contains { $0.value == "9" })205 #expect(finding.evidence.contains { $0.value == "3" })206 let (fewer, fewerReport) = try findings(buildZip(entries, declaredCount: 1))207 #expect(fewerReport.parsedEntryCount == 3)208 #expect(fewer.findings.has("file.zip.entryCountMismatch"))209 let (honest, _) = try findings(buildZip(entries))210 #expect(!honest.findings.has("file.zip.entryCountMismatch"))211 }212213 @Test("The walk stops at the entry limit")214 func entryLimit() throws {215 let entries = (0..<60).map { ZipEntrySpec("f\($0).txt", bytes("x")) }216 var options = InspectionOptions()217 options.maxArchiveEntries = 10218 options.maxReportedArchiveEntries = 4219 let (core, report) = try findings(buildZip(entries), options: options)220 #expect(report.parsedEntryCount == 10)221 #expect(report.entriesTruncated)222 #expect(report.entries.count == 4)223 #expect(core.findings.has("file.zip.tooManyEntries"))224 #expect(!core.findings.has("file.zip.entryCountMismatch"))225 }226227 @Test("A directory of 150 000 entries is cut at 100 000 in bounded time")228 func manyEntries() throws {229 var directory: [UInt8] = []230 directory.reserveCapacity(150_000 * 56)231 for index in 0..<150_000 {232 let name = bytes(String(format: "f%08d", index))233 directory += [0x50, 0x4B, 0x01, 0x02] + le16(0x031E) + le16(20) + le16(0x0800) + le16(8) + le16(0) + le16(0)234 directory += le32(0) + le32(10) + le32(20) + le16(UInt16(name.count)) + le16(0) + le16(0) + le16(0) + le16(0) + le32(0) + le32(UInt32(index))235 directory += name236 }237 let data = directory + [0x50, 0x4B, 0x05, 0x06] + le16(0) + le16(0) + le16(0xFFFE) + le16(0xFFFE) + le32(UInt32(directory.count)) + le32(0) + le16(0)238 let start = Date()239 let inspection = try inspectZip(data)240 let elapsed = Date().timeIntervalSince(start)241 print("ZIP central directory: 100 000 entries of \(data.count / 1_000_000) MB read in \(String(format: "%.2f", elapsed)) s")242 #expect(inspection.report.parsedEntryCount == 100_000)243 #expect(inspection.report.entriesTruncated)244 #expect(inspection.report.entries.count == 500)245 #expect(inspection.report.totalUncompressedSize == 2_000_000)246 #expect(elapsed < 30)247 }248249 @Test("Office documents: macros, required part, embedded and linked parts")250 func office() throws {251 let macro = buildZip([ZipEntrySpec("[Content_Types].xml", bytes("<Types/>")), ZipEntrySpec("word/document.xml", bytes("<d/>")), ZipEntrySpec("word/vbaProject.bin", zeros(64))])252 let hidden = try inspectBytes(macro, name: "contract.docx")253 expectHonest(hidden.findings)254 #expect(hidden.type == .docx)255 #expect(hidden.archive?.containsOfficeMacros == true)256 let finding = try #require(hidden.findings.with("file.ooxml.macrosInMacroFreeExtension"))257 #expect(finding.severity == .high)258 #expect(finding.notInspected.isEmpty == false)259 let declared = try inspectBytes(macro, name: "contract.docm")260 #expect(declared.findings.with("file.ooxml.macros")?.severity == .moderate)261 #expect(!declared.findings.has("file.ooxml.macrosInMacroFreeExtension"))262263 let linked = buildZip([264 ZipEntrySpec("[Content_Types].xml", bytes("<Types/>")), ZipEntrySpec("xl/workbook.xml", bytes("<w/>")),265 ZipEntrySpec("xl/externalLinks/externalLink1.xml", bytes("<e/>")), ZipEntrySpec("xl/embeddings/oleObject1.bin", zeros(8)),266 ])267 let workbook = try inspectBytes(linked, name: "budget.xlsx")268 #expect(workbook.archive?.externalLinkPartCount == 1)269 #expect(workbook.archive?.embeddedObjectPartCount == 1)270 #expect(workbook.findings.has("file.ooxml.external"))271272 let renamed = try inspectBytes(Samples.plainZip, name: "report.xlsx")273 #expect(renamed.findings.has("file.ooxml.contentTypes"))274 let good = try inspectBytes(Samples.docx, name: "letter.docx")275 #expect(!good.findings.contains { $0.state == .attention })276 }277278 @Test("Data placed before the archive is detected")279 func prependedData() throws {280 let stub = Samples.pe + zeros(100)281 let (core, report) = try findings(bytes("PK\u{3}\u{4}") + zeros(996) + Samples.plainZip, name: "sfx.zip")282 #expect(report.prependedByteCount == 1000)283 #expect(report.parsedEntryCount == 3)284 #expect(core.findings.has("file.zip.prepended"))285 // A program followed by an archive is detected as the program.286 let sfx = try inspectBytes(stub + Samples.plainZip, name: "setup.exe")287 #expect(sfx.type == .pe)288 }289290 @Test("Inconsistent sizes and archive comment")291 func sizesAndComment() throws {292 let data = buildZip([ZipEntrySpec("a.bin", zeros(4), declaredCompressed: 0, declaredUncompressed: 500), ZipEntrySpec("b.bin", zeros(4), declaredCompressed: 4, declaredUncompressed: 9)], comment: bytes("made by hand PK\u{5}\u{6} trap"))293 let (core, report) = try findings(data)294 #expect(report.isWellFormed)295 #expect(report.inconsistentSizeCount == 2)296 #expect(core.findings.has("file.zip.sizes"))297 }298299 @Test("Entry names with hidden characters are shown visibly")300 func hostileNames() throws {301 let (_, report) = try findings(buildZip([ZipEntrySpec("evil\u{202E}gpj.exe", zeros(1)), ZipEntrySpec(String(repeating: "A", count: 5000), zeros(1))]))302 #expect(report.entries[0].path == "evil<U+202E>gpj.exe")303 #expect(report.entries[1].path.count == 301)304 // Not valid UTF-8: decoded without failing.305 var raw = buildZip([ZipEntrySpec("abcd.txt", zeros(1))])306 let index = try #require(raw.lastIndex(of: bytes("abcd.txt")))307 raw[index] = 0xE9308 let second = try inspectZip(raw)309 #expect(second.report.entries.first?.path == "ébcd.txt")310 }311312 // MARK: TAR and GZIP313314 @Test("TAR header walk: names, sizes, links, traversal")315 func tar() throws {316 let data = buildTar([317 ("readme.txt", bytes("hello world"), "0"), ("dir/", [], "5"), ("dir/tool.sh", bytes("#!/bin/sh\n"), "0"),318 ("../outside.txt", bytes("x"), "0"), ("link", [], "2"), ("big.bin", [UInt8](repeating: 7, count: 1500), "0"),319 ])320 let (core, report) = try findings(data, name: "bundle.tar")321 #expect(core.type == .tar)322 #expect(report.format == .tar)323 #expect(report.isWellFormed)324 #expect(report.parsedEntryCount == 6)325 #expect(report.totalUncompressedSize == 11 + 10 + 1 + 1500)326 #expect(report.pathTraversalCount == 1)327 #expect(report.symbolicLinkCount == 1)328 #expect(report.executableCount == 1)329 #expect(core.findings.with("file.tar.traversal")?.severity == .high)330 #expect(core.findings.has("file.tar.links"))331 }332333 @Test("TAR: GNU long names, truncation and bad checksums")334 func tarEdgeCases() throws {335 let longName = String(repeating: "d/", count: 80) + "file.txt"336 var data = tarHeader(name: "././@LongLink", size: longName.utf8.count + 1, type: "L")337 data += bytes(longName) + zeros(512 - (longName.utf8.count % 512))338 data += tarHeader(name: "truncated-name", size: 3) + bytes("abc") + zeros(509) + zeros(1024)339 let long = try TarInspector.inspect(source: DataByteSource(data), options: .init())340 #expect(long.parsedEntryCount == 1)341 #expect(long.entries.first?.path == longName)342343 let whole = buildTar([("a.txt", [UInt8](repeating: 1, count: 2000), "0"), ("b.txt", bytes("b"), "0")])344 let cut = try TarInspector.inspect(source: DataByteSource(Array(whole.prefix(1000))), options: .init())345 #expect(!cut.isWellFormed)346 #expect(cut.parsedEntryCount == 1)347 var corrupt = whole348 corrupt[10] = 0x7A349 let (core, bad) = try findings(corrupt, name: "x.tar")350 #expect(!bad.isWellFormed)351 #expect(core.findings.has("file.tar.malformed"))352353 // A size field claiming 8 exabytes must not overflow.354 var huge = tarHeader(name: "huge", size: 0)355 huge.replaceSubrange(124..<136, with: [0x80, 0, 0, 0, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF])356 huge.replaceSubrange(148..<156, with: [UInt8](repeating: 0x20, count: 8))357 let sum = String(huge.reduce(0) { $0 + Int($1) }, radix: 8)358 huge.replaceSubrange(148..<156, with: bytes(String(repeating: "0", count: 6 - sum.count) + sum + "\u{0} "))359 let overflow = try TarInspector.inspect(source: DataByteSource(huge + zeros(1024)), options: .init())360 #expect(!overflow.isWellFormed)361 }362363 @Test("GZIP: original name, declared size and ratio")364 func gzip() throws {365 let header: [UInt8] = [0x1F, 0x8B, 0x08, 0x08, 0, 0, 0, 0, 0, 3] + bytes("installer.exe\u{0}")366 let data = header + zeros(40) + le32(0xDEAD_BEEF) + le32(3_000_000_000)367 let (core, report) = try findings(data, name: "installer.gz")368 #expect(core.type == .gzip)369 #expect(report.originalName == "installer.exe")370 #expect(report.totalUncompressedSize == 3_000_000_000)371 #expect(report.totalCompressedSize == UInt64(data.count))372 let ratio = try #require(core.findings.with("file.gzip.ratio"))373 #expect(ratio.confidence == .heuristic)374 #expect(ratio.severity == .moderate)375 #expect(core.findings.has("file.gzip.originalName"))376377 let modest = [0x1F, 0x8B, 0x08, 0x00, 0, 0, 0, 0, 0, 3] + zeros(100) + le32(0) + le32(400)378 let (plain, plainReport) = try findings(modest, name: "log.gz")379 #expect(plainReport.originalName == nil)380 #expect(!plain.findings.contains { $0.state == .attention })381 let short = try GzipInspector.inspect(source: DataByteSource([0x1F, 0x8B, 0x08]))382 #expect(!short.isWellFormed)383 }384}385