Repository Public License
1// ORIX Convert — Packages/ORIXConvertCore/Tests/ConvertCoreTests/DetectorTextTests.swift2// Description: FileTypeDetector on text: encodings, textual formats, extension refinement, and fuzz-style3// robustness (random bytes, truncations and mutations of every fixture, hostile containers).4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Compression6import Foundation7import Testing8@testable import ConvertCore910private func detect(_ text: String, _ name: String? = nil) -> DetectionResult {11 FileTypeDetector.detect(data: Data(text.utf8), fileName: name)12}1314private func detect(_ bytes: [UInt8], _ name: String? = nil) -> DetectionResult {15 FileTypeDetector.detect(data: Data(bytes), fileName: name)16}1718@Suite struct DetectorEncodingTests {19 @Test func asciiAndUTF8() {20 #expect(detect("plain words\n").textEncoding == .ascii)21 #expect(detect("café crème — déjà vu\n").textEncoding == .utf8)22 #expect(detect([0xEF, 0xBB, 0xBF] + ascii("with bom\n")).textEncoding == .utf8BOM)23 }2425 @Test func utf16WithAndWithoutBOM() {26 let le = Array("hello, world\n".data(using: .utf16LittleEndian)!)27 let be = Array("hello, world\n".data(using: .utf16BigEndian)!)28 #expect(detect([0xFF, 0xFE] + le).textEncoding == .utf16LE)29 #expect(detect([0xFE, 0xFF] + be).textEncoding == .utf16BE)30 #expect(detect(le).textEncoding == .utf16LE)31 #expect(detect(be).textEncoding == .utf16BE)32 #expect(detect([0xFF, 0xFE] + le).format == .txt)33 }3435 @Test func utf32BOMs() {36 #expect(detect([0xFF, 0xFE, 0x00, 0x00] + Array("hi\n".data(using: .utf32LittleEndian)!)).textEncoding == .utf32LE)37 #expect(detect([0x00, 0x00, 0xFE, 0xFF] + Array("hi\n".data(using: .utf32BigEndian)!)).textEncoding == .utf32BE)38 }3940 @Test func latin1Fallback() {41 let result = detect(ascii("Le caf") + [0xE9] + ascii(" est pr") + [0xEA] + ascii("t depuis longtemps.\n"))42 #expect(result.textEncoding == .latin1 && result.format == .txt)43 }4445 @Test func formatsAreRecognisedThroughUTF16() {46 let json = Array("{\"a\": 1, \"b\": [true, null]}".data(using: .utf16LittleEndian)!)47 #expect(detect([0xFF, 0xFE] + json).format == .json)48 let xml = Array("<?xml version=\"1.0\"?><svg xmlns=\"http://www.w3.org/2000/svg\"/>".data(using: .utf16BigEndian)!)49 #expect(detect([0xFE, 0xFF] + xml).format == .svg)50 }5152 @Test func utf8CutInsideACharacterIsStillText() {53 // 70 000 bytes of two-byte characters: the 64 KiB prefix boundary falls inside one of them.54 let text = "x" + String(repeating: "é", count: 35_000)55 let result = detect(text)56 #expect(result.format == .txt && result.textEncoding == .utf8)57 }5859 @Test func binaryWithControlBytesIsNotText() {60 #expect(detect([0x01, 0x02, 0x03, 0x04, 0x41, 0x42, 0x10, 0x11] + zeros(4)).textEncoding == nil)61 }62}6364@Suite struct DetectorMarkupTests {65 @Test func svgWithAndWithoutDeclaration() {66 #expect(detect("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!-- made by hand -->\n<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"1\"/>").format == .svg)67 #expect(detect("<svg viewBox=\"0 0 1 1\"><rect/></svg>").format == .svg)68 #expect(detect("<svg:svg xmlns:svg=\"http://www.w3.org/2000/svg\"/>").format == .svg)69 let doctype = "<?xml version=\"1.0\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<svg/>"70 #expect(detect(doctype).format == .svg)71 }7273 @Test func html() {74 #expect(detect("<!DOCTYPE html>\n<html><head><title>t</title></head><body></body></html>").format == .html)75 #expect(detect("<!doctype HTML>\n<p>x</p>").format == .html)76 #expect(detect("<html lang=\"en\"><body>x</body></html>").format == .html)77 let fragment = detect("<div class=\"a\"><p>hello</p></div>")78 #expect(fragment.format == .html && fragment.confidence == .likely)79 #expect(detect("<?xml version=\"1.0\"?><html xmlns=\"http://www.w3.org/1999/xhtml\"><body/></html>").format == .html)80 }8182 @Test func xmlFamily() {83 #expect(detect("<?xml version=\"1.0\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\"><Document/></kml>").format == .kml)84 #expect(detect("<?xml version=\"1.0\"?>\n<gpx version=\"1.1\" creator=\"x\"><trk/></gpx>").format == .gpx)85 #expect(detect("<?xml version=\"1.0\"?>\n<COLLADA xmlns=\"http://www.collada.org/2005/11/COLLADASchema\" version=\"1.4.1\"/>").format == .dae)86 let plist = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n<plist version=\"1.0\"><dict/></plist>"87 #expect(detect(plist).format == .plist)88 let generic = detect("<?xml version=\"1.0\"?>\n<catalog><book id=\"1\"/></catalog>")89 #expect(generic.format == .xml && generic.confidence == .certain)90 #expect(detect("<catalog><book id=\"1\"/></catalog>").format == .xml)91 #expect(detect("<?xml version=\"1.0\"?>\n<!DOCTYPE note [ <!ENTITY a \"b\"> ]>\n<note>&a;</note>").format == .xml)92 }9394 @Test func specificXMLNamedGenericallyIsNotAMismatch() {95 #expect(!detect("<svg viewBox=\"0 0 1 1\"/>", "drawing.xml").mismatch)96 #expect(detect("<?xml version=\"1.0\"?><catalog/>", "drawing.svg").mismatch)97 }9899 @Test func lessThanInProseIsNotMarkup() {100 #expect(detect("<3 this is a heart and nothing else\n").format == .txt)101 #expect(detect("a < b and b > c\n").format == .txt)102 }103104 @Test func markdown() {105 #expect(detect("# Title\n\nSome **bold** text and a [link](https://example.org).\n\n- one\n- two\n").format == .markdown)106 #expect(detect("Intro paragraph.\n\n```swift\nlet a = 1\n```\n\n> quoted\n").format == .markdown)107 #expect(detect("---\ntitle: Post\ndate: 2026-01-01\n---\n\n# Heading\n\nProse with **emphasis** here.\n").format == .markdown, "front matter then prose")108 }109110 @Test func sourceCodeAndConfigurationAreNotMarkdown() {111 #expect(detect("#!/bin/sh\n\n# Build\n\nmake **all**\n").format == .txt, "shebang")112 #expect(detect("# comment one\n# comment two\nimport os\n\ndef f(**kwargs):\n return self.__init__(**kwargs)\n").format == .txt)113 #expect(detect("/*\n * Header\n * More\n */\nint main(void) { return 0; }\n").format == .txt, "comment blocks are not lists")114 #expect(detect("# Sample configuration\n# for the demo\nHost *\n Port 22\n").format == .txt, "comment lines are not headings")115 let slides = detect("<section class=\"cover\">\n\n# Title\n\n</section>\n", "deck.md")116 #expect(slides.format == .markdown && !slides.mismatch, "Markdown may start with HTML")117 }118119 @Test func latex() {120 let full = detect("\\documentclass{article}\n\\begin{document}\nHello\n\\end{document}\n")121 #expect(full.format == .latex && full.confidence == .certain)122 let partial = detect("\\section{Results}\nWe observe that $x^2$ grows.\n")123 #expect(partial.format == .latex && partial.confidence == .likely)124 }125126 @Test func reStructuredText() {127 #expect(detect("Title\n=====\n\n.. note::\n\n Remember this.\n").format == .rst)128 #expect(detect("Guide\n=====\n\nSee :ref:`install` first.\n").format == .rst)129 #expect(detect("Chapter\n=======\n\nText.\n\nSection\n-------\n\nMore text.\n", "doc.rst").format == .rst)130 #expect(detect("Chapter\n=======\n\nText.\n\nSection\n-------\n\nMore text.\n").format == .rst)131 }132133 @Test func rtfAsText() {134 #expect(detect(" {\\rtf1 x}").format == .rtf)135 }136}137138@Suite struct DetectorStructuredTextTests {139 @Test func jsonVariants() {140 let object = detect("{\"name\": \"x\", \"values\": [1, 2, 3]}")141 #expect(object.format == .json && object.confidence == .certain)142 #expect(detect("[1, 2, 3]").format == .json)143 #expect(detect("{\n \"a\": {\n \"b\": 1\n }\n}\n").format == .json)144 #expect(detect("{not json at all}").format != .json)145 }146147 @Test func jsonLines() {148 let result = detect("{\"a\": 1}\n{\"a\": 2}\n{\"a\": 3}\n")149 #expect(result.format == .jsonl && result.confidence == .certain)150 #expect(detect("[1, 2]\n[3, 4]\n").format == .jsonl)151 // One record is valid JSON; the name can still say JSON Lines.152 #expect(detect("{\"a\": 1}\n").format == .json)153 let single = detect("{\"a\": 1}\n", "events.jsonl")154 #expect(single.format == .jsonl && !single.mismatch)155 }156157 @Test func geoJSON() {158 #expect(detect("{\"type\": \"FeatureCollection\", \"features\": []}").format == .geojson)159 #expect(detect("{\"type\": \"Feature\", \"geometry\": null, \"properties\": {}}").format == .geojson)160 #expect(detect("{\"type\": \"Point\", \"coordinates\": [1.0, 2.0]}").format == .geojson)161 #expect(detect("{\"type\": \"Point\", \"x\": 1}").format == .json, "a 'type' key alone is not GeoJSON")162 #expect(!detect("{\"type\": \"FeatureCollection\", \"features\": []}", "map.json").mismatch)163 }164165 @Test func gltf() {166 #expect(detect("{\"asset\": {\"version\": \"2.0\"}, \"scenes\": [], \"nodes\": []}").format == .gltf)167 #expect(detect("{\"asset\": \"not an object\"}").format == .json)168 }169170 @Test func largeJSONIsClassifiedFromItsPrefix() {171 let features = (0..<4000).map { "{\"type\":\"Feature\",\"geometry\":{\"type\":\"Point\",\"coordinates\":[\($0),0]},\"properties\":{}}" }.joined(separator: ",")172 let big = "{\"type\":\"FeatureCollection\",\"features\":[" + features + "]}"173 #expect(big.utf8.count > FileTypeDetector.prefixLength)174 let result = detect(big)175 #expect(result.format == .geojson && result.confidence == .likely)176 let rows = "[" + (0..<20_000).map { "{\"id\":\($0)}" }.joined(separator: ",") + "]"177 #expect(detect(rows).format == .json)178 // The "type" key comes after the prefix: generic JSON, and the extension is trusted.179 let late = "{\"features\":[" + features + "],\"type\":\"FeatureCollection\"}"180 #expect(detect(late).format == .json)181 let named = detect(late, "big.geojson")182 #expect(named.format == .geojson && !named.mismatch)183 }184185 @Test func yaml() {186 #expect(detect("---\nname: test\nitems:\n - one\n - two\n").format == .yaml)187 #expect(detect("name: test\nversion: 3\nenabled: true\n").format == .yaml)188 #expect(detect("%YAML 1.2\n---\nkey: value\n").format == .yaml)189 #expect(detect("Dear team: here is the plan.\nIt has no structure at all, just prose.\n").format == .txt)190 }191192 @Test func toml() {193 #expect(detect("[package]\nname = \"demo\"\nversion = \"0.1.0\"\n\n[dependencies]\nserde = { version = \"1\" }\n").format == .toml)194 #expect(detect("title = \"x\"\ncount = 3\nenabled = true\n").format == .toml)195 #expect(detect("KEY=value\nOTHER=thing\n").format != .toml, "bare .env values are not TOML")196 #expect(detect("[ req ]\ndefault_bits = 2048\ndir = ./demoCA\n").format != .toml, "INI files with bare values are not TOML")197 }198199 @Test func csvVersusTSV() {200 #expect(detect("id,name,price\n1,apple,0.5\n2,pear,0.75\n").format == .csv)201 #expect(detect("id\tname\tprice\n1\tapple\t0.5\n2\tpear\t0.75\n").format == .tsv)202 #expect(detect("id;name;price\n1;apple;0,5\n2;pear;0,75\n").format == .csv)203 #expect(detect("id,name\n1,\"Smith, John\"\n2,\"Doe, Jane\"\n").format == .csv, "quoted delimiters do not count")204 #expect(detect("id,note\n1,\"line one\nline two\"\n2,\"x\"\n").format == .csv, "quoted line breaks are joined")205 #expect(detect("a,b,c\n1,2\n3,4,5,6\n").format == .txt, "inconsistent columns are not CSV")206 #expect(detect("let a = 1;\nlet b = 2;\nprint(a + b);\n").format == .txt, "statement terminators are not column separators")207 #expect(detect("a,b\n\"x\"\"y\",2\n\"z\",3\n").format == .csv, "doubled quotes are escapes")208 #expect(detect("\" Vim syntax file\n\" Language:\tDemo\n\" Maintainer:\tNobody\n").format == .txt, "a stray quote does not swallow the following lines")209 #expect(detect("\tif x\n\t\tcall y()\n\tendif\n\treturn z\n").format == .txt, "indentation is not a column separator")210 // Tabs win over commas inside cells.211 #expect(detect("name\tnote\nx\ta, b\ny\tc, d\n").format == .tsv)212 }213214 @Test func subtitles() {215 #expect(detect("1\n00:00:01,000 --> 00:00:02,500\nHello\n\n2\n00:00:03,000 --> 00:00:04,000\nWorld\n").format == .srt)216 #expect(detect("WEBVTT\n\n00:00:01.000 --> 00:00:02.500\nHello\n").format == .vtt)217 #expect(detect("[Script Info]\nScriptType: v4.00+\n\n[V4+ Styles]\nFormat: Name\n\n[Events]\n").format == .ass)218 #expect(detect("[Script Info]\nScriptType: v4.00\n\n[V4 Styles]\nFormat: Name\n").format == .ssa)219 #expect(detect("{0}{25}Hello\n{30}{60}World|second line\n").format == .sub)220 #expect(detect("1\nThis is just a numbered note.\n").format == .txt)221 }222223 @Test func textual3DFormats() {224 #expect(detect("# cube\nv 0 0 0\nv 1 0 0\nv 1 1 0\nvn 0 0 1\nf 1 2 3\n").format == .obj)225 #expect(detect("#usda 1.0\n(\n defaultPrim = \"World\"\n)\n").format == .usda)226 #expect(detect("solid cube\n facet normal 0 0 1\n outer loop\n vertex 0 0 0\n endloop\n endfacet\nendsolid cube\n").format == .stl)227 #expect(detect("ply\nformat ascii 1.0\nelement vertex 0\nend_header\n").format == .ply)228 }229230 @Test func plainText() {231 let result = detect("Just a few words without any structure.\nAnother line follows here.\n")232 #expect(result.format == .txt && result.confidence == .likely)233 #expect(detect(" \n\n").format == .txt)234 }235}236237@Suite struct DetectorRefinementTests {238 @Test func looseTextFollowsTheExtension() {239 let md = detect("Just a few words.\n", "README.md")240 #expect(md.format == .markdown && !md.mismatch)241 let csv = detect("single line, with one comma\n", "one.csv")242 #expect(csv.format == .csv && !csv.mismatch)243 let tex = detect("Some included fragment without commands.\n", "part.tex")244 #expect(tex.format == .latex && !tex.mismatch)245 // Markdown saved as .txt stays plain text for the user.246 #expect(detect("# Title\n\n**bold** and [a](b)\n", "notes.txt").format == .txt)247 }248249 @Test func structuredNamesAreNotSatisfiedByProse() {250 #expect(detect("hello there\n", "data.json").mismatch)251 #expect(detect("hello there\n", "movie.srt").mismatch)252 #expect(detect("hello there\n", "page.html").mismatch)253 }254255 @Test func structuredContentUnderATextNameIsKept() {256 let result = detect("{\"a\": 1}", "dump.txt")257 #expect(result.format == .json && !result.mismatch)258 }259260 @Test func typeScriptIsNotATransportStream() {261 let result = detect("export const answer: number = 42;\nconsole.log(answer);\n", "index.ts")262 #expect(result.format == .txt && result.mismatch)263 }264265 @Test func unknownExtensionsNeverMismatch() {266 #expect(!detect("{\"a\": 1}", "file.weird").mismatch)267 #expect(detect("{\"a\": 1}", "file.weird").format == .json)268 }269}270271/// Deterministic generator so failures are reproducible.272private struct SplitMix64: RandomNumberGenerator {273 var state: UInt64274 mutating func next() -> UInt64 {275 state &+= 0x9E37_79B9_7F4A_7C15276 var z = state277 z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9278 z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB279 return z ^ (z >> 31)280 }281}282283@Suite struct DetectorFuzzTests {284 static let corpus: [[UInt8]] = {285 let types = ascii("<?xml version=\"1.0\"?><Types/>")286 return [287 Fixtures.png, Fixtures.apng, Fixtures.mp3Frames, Fixtures.tar(), Fixtures.gzip(Fixtures.tar()),288 compress(Fixtures.tar(), COMPRESSION_LZMA),289 Fixtures.zip([("[Content_Types].xml", types), ("word/document.xml", ascii("<w/>"))], deflated: ["[Content_Types].xml"]),290 Fixtures.zip([("mimetype", ascii("application/epub+zip")), ("content.opf", ascii("<x/>"))]),291 Fixtures.ole2(streams: ["WordDocument"]),292 Fixtures.ftyp("M4A ", compatible: ["isom"]) + be32(60) + ascii("moov") + be32(52) + ascii("stsd") + be32(0) + be32(1) + be32(36) + ascii("alac") + zeros(28),293 [0x49, 0x49, 0x2A, 0x00] + le32(8) + le16(2) + le16(256) + le16(3) + le32(1) + le32(1) + le16(0xC612) + le16(1) + le32(4) + le32(1) + le32(0),294 ascii("OggS") + zeros(22) + [1, 19] + ascii("OpusHead") + zeros(11),295 [0x1A, 0x45, 0xDF, 0xA3, 0x9F, 0x42, 0x86, 0x81, 0x01, 0x42, 0x82, 0x84] + ascii("webm"),296 ascii("ID3") + [3, 0, 0, 0, 0, 0, 10] + zeros(10) + Fixtures.mp3Frames,297 ascii("SQLite format 3\0") + zeros(84),298 ascii("P6\n# c\n2 2\n255\n") + zeros(12),299 zeros(80) + le32(2) + zeros(100),300 ascii("{\"type\": \"FeatureCollection\", \"features\": [{\"type\": \"Feature\"}]}"),301 ascii("<?xml version=\"1.0\"?>\n<!DOCTYPE svg [ <!ENTITY a \"b\"> ]>\n<svg xmlns=\"http://www.w3.org/2000/svg\"/>"),302 ascii("1\n00:00:01,000 --> 00:00:02,500\nHello\n"),303 ascii("---\nname: x\nlist:\n - a\n"),304 ascii("[package]\nname = \"demo\"\n"),305 ascii("id,name\n1,\"a, b\"\n2,\"c\nd\"\n"),306 ascii("Title\n=====\n\n.. note::\n x\n"),307 Array("héllo wörld\n".data(using: .utf16LittleEndian)!),308 [0xEF, 0xBB, 0xBF] + ascii("# Title\n\n**b** [a](b)\n"),309 ]310 }()311312 @Test func randomBytesNeverCrash() {313 var rng = SplitMix64(state: 0x0E1_C0DE)314 for round in 0..<3000 {315 let length = round % 7 == 0 ? Int.random(in: 0...70_000, using: &rng) : Int.random(in: 0...600, using: &rng)316 var bytes = [UInt8](repeating: 0, count: length)317 for i in bytes.indices { bytes[i] = UInt8.random(in: 0...255, using: &rng) }318 let result = FileTypeDetector.detect(data: Data(bytes), fileName: round % 3 == 0 ? "x.zip" : nil)319 #expect(result.format != nil || result.confidence == .extensionOnly)320 }321 }322323 @Test func randomBytesBehindEveryKnownMagicNeverCrash() {324 var rng = SplitMix64(state: 42)325 let magics: [[UInt8]] = [326 [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], [0xFF, 0xD8, 0xFF], ascii("GIF89a"), [0x49, 0x49, 0x2A, 0x00], [0x4D, 0x4D, 0x00, 0x2A],327 ascii("BM"), ascii("RIFF"), [0, 0, 0, 24] + ascii("ftyp"), [0, 0, 0, 0] + ascii("ftyp"), [0xFF, 0xFF, 0xFF, 0xFF] + ascii("ftypM4A "),328 [0, 0, 0, 1] + ascii("ftypM4A "), [0x1A, 0x45, 0xDF, 0xA3], ascii("OggS"), ascii("ID3"), [0xFF, 0xFB], [0xFF, 0xF1], ascii("FORM"),329 [0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11, 0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C], [0x47], ascii("8BPS"), [0, 0, 1, 0],330 ascii("icns"), ascii("P6 "), ascii("%PDF-"), [0x50, 0x4B, 0x03, 0x04], [0x50, 0x4B, 0x05, 0x06], [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1],331 [0x1F, 0x8B, 0x08], [0x1F, 0x8B, 0x08, 0x1C], ascii("BZh9"), [0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00], ascii("SQLite format 3\0"), ascii("PAR1"),332 [0x89, 0x48, 0x44, 0x46, 0x0D, 0x0A, 0x1A, 0x0A], [0, 1, 0, 0], ascii("glTF"), ascii("ply\n"), ascii("solid "), ascii("<?xml"), ascii("<"),333 ascii("{"), ascii("["), ascii("---\n"), [0xFF, 0xFE], [0xFE, 0xFF], [0xEF, 0xBB, 0xBF], ascii("{\\rtf"), ascii("1\n"), ascii("WEBVTT"),334 ]335 for magic in magics {336 for _ in 0..<60 {337 let length = Int.random(in: 0...2048, using: &rng)338 var bytes = magic339 for _ in 0..<length { bytes.append(UInt8.random(in: 0...255, using: &rng)) }340 _ = FileTypeDetector.detect(data: Data(bytes), fileName: "f.bin")341 }342 }343 }344345 @Test func everyTruncationOfEveryFixtureIsHandled() {346 for fixture in Self.corpus {347 let step = max(1, fixture.count / 700)348 for length in stride(from: 0, through: fixture.count, by: step) {349 _ = FileTypeDetector.detect(data: Data(fixture[0..<length]), fileName: nil)350 }351 }352 }353354 @Test func mutatedFixturesAreHandled() {355 var rng = SplitMix64(state: 7)356 for fixture in Self.corpus where !fixture.isEmpty {357 for _ in 0..<150 {358 var bytes = fixture359 for _ in 0..<Int.random(in: 1...6, using: &rng) {360 let index = Int.random(in: 0..<bytes.count, using: &rng)361 bytes[index] = [0x00, 0xFF, 0x7F, 0x80, UInt8.random(in: 0...255, using: &rng)].randomElement(using: &rng) ?? 0362 }363 _ = FileTypeDetector.detect(data: Data(bytes), fileName: "m.dat")364 }365 }366 }367368 @Test func hostileZipDirectoriesAreBounded() {369 // End record pointing far outside the file, with an absurd entry count.370 let bogus = [0x50, 0x4B, 0x03, 0x04] + zeros(26) + [0x50, 0x4B, 0x05, 0x06] + le16(0) + le16(0) + le16(0xFFFF) + le16(0xFFFF) + le32(0xFFFF_FFF0) + le32(0x7FFF_FFFF) + le16(0) as [UInt8]371 #expect(detect(bogus).format == .zip)372 // ZIP64 locator pointing nowhere.373 let zip64 = [0x50, 0x4B, 0x03, 0x04] + zeros(26) + [0x50, 0x4B, 0x06, 0x07] + le32(0) + le32(0xFFFF_FFFF) + le32(0x7FFF_FFFF) + le32(1)374 + [0x50, 0x4B, 0x05, 0x06] + le16(0) + le16(0) + le16(0xFFFF) + le16(0xFFFF) + le32(0xFFFF_FFFF) + le32(0xFFFF_FFFF) + le16(0) as [UInt8]375 #expect(detect(zip64).format == .zip)376 // Local header declaring a name longer than the file.377 let longName = [0x50, 0x4B, 0x03, 0x04] + zeros(22) + le16(0xFFFF) + le16(0xFFFF) + ascii("abc") as [UInt8]378 #expect(detect(longName).format == .zip)379 // Central directory entry with lengths running past the end.380 var cut = Fixtures.zip([("a.txt", ascii("a"))])381 if let cd = cut.find([0x50, 0x4B, 0x01, 0x02]) { cut[cd + 28] = 0xFF; cut[cd + 29] = 0xFF }382 #expect(detect(cut).format == .zip)383 }384385 @Test func hostileOLE2ChainsTerminate() {386 var looping = Fixtures.ole2(streams: ["WordDocument"])387 // FAT entry of the directory sector points to itself.388 looping.replaceSubrange((512 + 4)..<(512 + 8), with: le32(1))389 #expect(detect(looping).format == .doc)390 var absurd = Fixtures.ole2(streams: ["Workbook"])391 absurd.replaceSubrange(48..<52, with: le32(0x7FFF_FFF0)) // first directory sector far away392 #expect(detect(absurd).format == nil)393 var badShift = Fixtures.ole2(streams: ["Workbook"])394 badShift[30] = 31395 #expect(detect(badShift).format == nil)396 }397398 @Test func hostileBoxAndChunkLengths() {399 // PNG chunk length of 4 GiB.400 #expect(detect([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A] + be32(0xFFFF_FFFF) + ascii("IHDR") + zeros(8)).format == .png)401 // 64-bit box size close to Int64.max after the ftyp box.402 let huge = be32(16) + ascii("ftyp") + ascii("M4A ") + be32(0) + be32(1) + ascii("mdat") + [0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xF0] + zeros(8)403 #expect(detect(huge).format == .m4a)404 let negative = be32(16) + ascii("ftyp") + ascii("M4A ") + be32(0) + be32(1) + ascii("mdat") + [UInt8](repeating: 0xFF, count: 8)405 #expect(detect(negative).format == .m4a)406 // TIFF whose IFD offset points past the end.407 #expect(detect([0x49, 0x49, 0x2A, 0x00] + le32(0x7FFF_FFFF) + zeros(8)).format == .tiff)408 // ID3 tag claiming 256 MiB.409 #expect(detect(ascii("ID3") + [4, 0, 0, 0x7F, 0x7F, 0x7F, 0x7F] + zeros(64)).format == .mp3)410 // gzip header whose extra field runs past the end.411 #expect(detect([0x1F, 0x8B, 0x08, 0x04, 0, 0, 0, 0, 0, 3, 0xFF, 0xFF, 1, 2, 3]).format == .gz)412 // EBML DocType with a length beyond the data.413 #expect(detect([0x1A, 0x45, 0xDF, 0xA3, 0x42, 0x82, 0xFF, 0x77]).format == .mkv)414 }415416 @Test func deeplyNestedAndUnterminatedTextIsHandled() {417 #expect(detect(String(repeating: "[", count: 60_000)).format != nil)418 #expect(detect("{\"a\": \"" + String(repeating: "x", count: 100_000)).format != nil)419 #expect(detect("<?xml version=\"1.0\"" + String(repeating: " ", count: 70_000)).format != nil)420 #expect(detect("<!--" + String(repeating: "-", count: 70_000)).format != nil)421 #expect(detect(String(repeating: "\"", count: 9_999) + "\n" + String(repeating: ",", count: 5_000)).format != nil)422 #expect(detect(String(repeating: "- ", count: 30_000)).format != nil)423 }424}425