SPB Git

Repository Public License

DetectorTextTests.swift 24.3 KB 425 lines swift

1// ORIX Convert — Packages/ORIXConvertCore/Tests/ConvertCoreTests/DetectorTextTests.swift2// Description: FileTypeDetector on text: encodings, textual formats, extension refinement, and fuzz-style3//              robustness (random bytes, truncations and mutations of every fixture, hostile containers).4// Author: Simon-Pierre Boucher — contact@spboucher.ai5import Compression6import Foundation7import Testing8@testable import ConvertCore910private func detect(_ text: String, _ name: String? = nil) -> DetectionResult {11    FileTypeDetector.detect(data: Data(text.utf8), fileName: name)12}1314private func detect(_ bytes: [UInt8], _ name: String? = nil) -> DetectionResult {15    FileTypeDetector.detect(data: Data(bytes), fileName: name)16}1718@Suite struct DetectorEncodingTests {19    @Test func asciiAndUTF8() {20        #expect(detect("plain words\n").textEncoding == .ascii)21        #expect(detect("café crème — déjà vu\n").textEncoding == .utf8)22        #expect(detect([0xEF, 0xBB, 0xBF] + ascii("with bom\n")).textEncoding == .utf8BOM)23    }2425    @Test func utf16WithAndWithoutBOM() {26        let le = Array("hello, world\n".data(using: .utf16LittleEndian)!)27        let be = Array("hello, world\n".data(using: .utf16BigEndian)!)28        #expect(detect([0xFF, 0xFE] + le).textEncoding == .utf16LE)29        #expect(detect([0xFE, 0xFF] + be).textEncoding == .utf16BE)30        #expect(detect(le).textEncoding == .utf16LE)31        #expect(detect(be).textEncoding == .utf16BE)32        #expect(detect([0xFF, 0xFE] + le).format == .txt)33    }3435    @Test func utf32BOMs() {36        #expect(detect([0xFF, 0xFE, 0x00, 0x00] + Array("hi\n".data(using: .utf32LittleEndian)!)).textEncoding == .utf32LE)37        #expect(detect([0x00, 0x00, 0xFE, 0xFF] + Array("hi\n".data(using: .utf32BigEndian)!)).textEncoding == .utf32BE)38    }3940    @Test func latin1Fallback() {41        let result = detect(ascii("Le caf") + [0xE9] + ascii(" est pr") + [0xEA] + ascii("t depuis longtemps.\n"))42        #expect(result.textEncoding == .latin1 && result.format == .txt)43    }4445    @Test func formatsAreRecognisedThroughUTF16() {46        let json = Array("{\"a\": 1, \"b\": [true, null]}".data(using: .utf16LittleEndian)!)47        #expect(detect([0xFF, 0xFE] + json).format == .json)48        let xml = Array("<?xml version=\"1.0\"?><svg xmlns=\"http://www.w3.org/2000/svg\"/>".data(using: .utf16BigEndian)!)49        #expect(detect([0xFE, 0xFF] + xml).format == .svg)50    }5152    @Test func utf8CutInsideACharacterIsStillText() {53        // 70 000 bytes of two-byte characters: the 64 KiB prefix boundary falls inside one of them.54        let text = "x" + String(repeating: "é", count: 35_000)55        let result = detect(text)56        #expect(result.format == .txt && result.textEncoding == .utf8)57    }5859    @Test func binaryWithControlBytesIsNotText() {60        #expect(detect([0x01, 0x02, 0x03, 0x04, 0x41, 0x42, 0x10, 0x11] + zeros(4)).textEncoding == nil)61    }62}6364@Suite struct DetectorMarkupTests {65    @Test func svgWithAndWithoutDeclaration() {66        #expect(detect("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!-- made by hand -->\n<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"1\"/>").format == .svg)67        #expect(detect("<svg viewBox=\"0 0 1 1\"><rect/></svg>").format == .svg)68        #expect(detect("<svg:svg xmlns:svg=\"http://www.w3.org/2000/svg\"/>").format == .svg)69        let doctype = "<?xml version=\"1.0\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<svg/>"70        #expect(detect(doctype).format == .svg)71    }7273    @Test func html() {74        #expect(detect("<!DOCTYPE html>\n<html><head><title>t</title></head><body></body></html>").format == .html)75        #expect(detect("<!doctype HTML>\n<p>x</p>").format == .html)76        #expect(detect("<html lang=\"en\"><body>x</body></html>").format == .html)77        let fragment = detect("<div class=\"a\"><p>hello</p></div>")78        #expect(fragment.format == .html && fragment.confidence == .likely)79        #expect(detect("<?xml version=\"1.0\"?><html xmlns=\"http://www.w3.org/1999/xhtml\"><body/></html>").format == .html)80    }8182    @Test func xmlFamily() {83        #expect(detect("<?xml version=\"1.0\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\"><Document/></kml>").format == .kml)84        #expect(detect("<?xml version=\"1.0\"?>\n<gpx version=\"1.1\" creator=\"x\"><trk/></gpx>").format == .gpx)85        #expect(detect("<?xml version=\"1.0\"?>\n<COLLADA xmlns=\"http://www.collada.org/2005/11/COLLADASchema\" version=\"1.4.1\"/>").format == .dae)86        let plist = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\">\n<plist version=\"1.0\"><dict/></plist>"87        #expect(detect(plist).format == .plist)88        let generic = detect("<?xml version=\"1.0\"?>\n<catalog><book id=\"1\"/></catalog>")89        #expect(generic.format == .xml && generic.confidence == .certain)90        #expect(detect("<catalog><book id=\"1\"/></catalog>").format == .xml)91        #expect(detect("<?xml version=\"1.0\"?>\n<!DOCTYPE note [ <!ENTITY a \"b\"> ]>\n<note>&a;</note>").format == .xml)92    }9394    @Test func specificXMLNamedGenericallyIsNotAMismatch() {95        #expect(!detect("<svg viewBox=\"0 0 1 1\"/>", "drawing.xml").mismatch)96        #expect(detect("<?xml version=\"1.0\"?><catalog/>", "drawing.svg").mismatch)97    }9899    @Test func lessThanInProseIsNotMarkup() {100        #expect(detect("<3 this is a heart and nothing else\n").format == .txt)101        #expect(detect("a < b and b > c\n").format == .txt)102    }103104    @Test func markdown() {105        #expect(detect("# Title\n\nSome **bold** text and a [link](https://example.org).\n\n- one\n- two\n").format == .markdown)106        #expect(detect("Intro paragraph.\n\n```swift\nlet a = 1\n```\n\n> quoted\n").format == .markdown)107        #expect(detect("---\ntitle: Post\ndate: 2026-01-01\n---\n\n# Heading\n\nProse with **emphasis** here.\n").format == .markdown, "front matter then prose")108    }109110    @Test func sourceCodeAndConfigurationAreNotMarkdown() {111        #expect(detect("#!/bin/sh\n\n# Build\n\nmake **all**\n").format == .txt, "shebang")112        #expect(detect("# comment one\n# comment two\nimport os\n\ndef f(**kwargs):\n    return self.__init__(**kwargs)\n").format == .txt)113        #expect(detect("/*\n * Header\n * More\n */\nint main(void) { return 0; }\n").format == .txt, "comment blocks are not lists")114        #expect(detect("# Sample configuration\n# for the demo\nHost *\n    Port 22\n").format == .txt, "comment lines are not headings")115        let slides = detect("<section class=\"cover\">\n\n# Title\n\n</section>\n", "deck.md")116        #expect(slides.format == .markdown && !slides.mismatch, "Markdown may start with HTML")117    }118119    @Test func latex() {120        let full = detect("\\documentclass{article}\n\\begin{document}\nHello\n\\end{document}\n")121        #expect(full.format == .latex && full.confidence == .certain)122        let partial = detect("\\section{Results}\nWe observe that $x^2$ grows.\n")123        #expect(partial.format == .latex && partial.confidence == .likely)124    }125126    @Test func reStructuredText() {127        #expect(detect("Title\n=====\n\n.. note::\n\n   Remember this.\n").format == .rst)128        #expect(detect("Guide\n=====\n\nSee :ref:`install` first.\n").format == .rst)129        #expect(detect("Chapter\n=======\n\nText.\n\nSection\n-------\n\nMore text.\n", "doc.rst").format == .rst)130        #expect(detect("Chapter\n=======\n\nText.\n\nSection\n-------\n\nMore text.\n").format == .rst)131    }132133    @Test func rtfAsText() {134        #expect(detect("  {\\rtf1 x}").format == .rtf)135    }136}137138@Suite struct DetectorStructuredTextTests {139    @Test func jsonVariants() {140        let object = detect("{\"name\": \"x\", \"values\": [1, 2, 3]}")141        #expect(object.format == .json && object.confidence == .certain)142        #expect(detect("[1, 2, 3]").format == .json)143        #expect(detect("{\n  \"a\": {\n    \"b\": 1\n  }\n}\n").format == .json)144        #expect(detect("{not json at all}").format != .json)145    }146147    @Test func jsonLines() {148        let result = detect("{\"a\": 1}\n{\"a\": 2}\n{\"a\": 3}\n")149        #expect(result.format == .jsonl && result.confidence == .certain)150        #expect(detect("[1, 2]\n[3, 4]\n").format == .jsonl)151        // One record is valid JSON; the name can still say JSON Lines.152        #expect(detect("{\"a\": 1}\n").format == .json)153        let single = detect("{\"a\": 1}\n", "events.jsonl")154        #expect(single.format == .jsonl && !single.mismatch)155    }156157    @Test func geoJSON() {158        #expect(detect("{\"type\": \"FeatureCollection\", \"features\": []}").format == .geojson)159        #expect(detect("{\"type\": \"Feature\", \"geometry\": null, \"properties\": {}}").format == .geojson)160        #expect(detect("{\"type\": \"Point\", \"coordinates\": [1.0, 2.0]}").format == .geojson)161        #expect(detect("{\"type\": \"Point\", \"x\": 1}").format == .json, "a 'type' key alone is not GeoJSON")162        #expect(!detect("{\"type\": \"FeatureCollection\", \"features\": []}", "map.json").mismatch)163    }164165    @Test func gltf() {166        #expect(detect("{\"asset\": {\"version\": \"2.0\"}, \"scenes\": [], \"nodes\": []}").format == .gltf)167        #expect(detect("{\"asset\": \"not an object\"}").format == .json)168    }169170    @Test func largeJSONIsClassifiedFromItsPrefix() {171        let features = (0..<4000).map { "{\"type\":\"Feature\",\"geometry\":{\"type\":\"Point\",\"coordinates\":[\($0),0]},\"properties\":{}}" }.joined(separator: ",")172        let big = "{\"type\":\"FeatureCollection\",\"features\":[" + features + "]}"173        #expect(big.utf8.count > FileTypeDetector.prefixLength)174        let result = detect(big)175        #expect(result.format == .geojson && result.confidence == .likely)176        let rows = "[" + (0..<20_000).map { "{\"id\":\($0)}" }.joined(separator: ",") + "]"177        #expect(detect(rows).format == .json)178        // The "type" key comes after the prefix: generic JSON, and the extension is trusted.179        let late = "{\"features\":[" + features + "],\"type\":\"FeatureCollection\"}"180        #expect(detect(late).format == .json)181        let named = detect(late, "big.geojson")182        #expect(named.format == .geojson && !named.mismatch)183    }184185    @Test func yaml() {186        #expect(detect("---\nname: test\nitems:\n  - one\n  - two\n").format == .yaml)187        #expect(detect("name: test\nversion: 3\nenabled: true\n").format == .yaml)188        #expect(detect("%YAML 1.2\n---\nkey: value\n").format == .yaml)189        #expect(detect("Dear team: here is the plan.\nIt has no structure at all, just prose.\n").format == .txt)190    }191192    @Test func toml() {193        #expect(detect("[package]\nname = \"demo\"\nversion = \"0.1.0\"\n\n[dependencies]\nserde = { version = \"1\" }\n").format == .toml)194        #expect(detect("title = \"x\"\ncount = 3\nenabled = true\n").format == .toml)195        #expect(detect("KEY=value\nOTHER=thing\n").format != .toml, "bare .env values are not TOML")196        #expect(detect("[ req ]\ndefault_bits = 2048\ndir = ./demoCA\n").format != .toml, "INI files with bare values are not TOML")197    }198199    @Test func csvVersusTSV() {200        #expect(detect("id,name,price\n1,apple,0.5\n2,pear,0.75\n").format == .csv)201        #expect(detect("id\tname\tprice\n1\tapple\t0.5\n2\tpear\t0.75\n").format == .tsv)202        #expect(detect("id;name;price\n1;apple;0,5\n2;pear;0,75\n").format == .csv)203        #expect(detect("id,name\n1,\"Smith, John\"\n2,\"Doe, Jane\"\n").format == .csv, "quoted delimiters do not count")204        #expect(detect("id,note\n1,\"line one\nline two\"\n2,\"x\"\n").format == .csv, "quoted line breaks are joined")205        #expect(detect("a,b,c\n1,2\n3,4,5,6\n").format == .txt, "inconsistent columns are not CSV")206        #expect(detect("let a = 1;\nlet b = 2;\nprint(a + b);\n").format == .txt, "statement terminators are not column separators")207        #expect(detect("a,b\n\"x\"\"y\",2\n\"z\",3\n").format == .csv, "doubled quotes are escapes")208        #expect(detect("\" Vim syntax file\n\" Language:\tDemo\n\" Maintainer:\tNobody\n").format == .txt, "a stray quote does not swallow the following lines")209        #expect(detect("\tif x\n\t\tcall y()\n\tendif\n\treturn z\n").format == .txt, "indentation is not a column separator")210        // Tabs win over commas inside cells.211        #expect(detect("name\tnote\nx\ta, b\ny\tc, d\n").format == .tsv)212    }213214    @Test func subtitles() {215        #expect(detect("1\n00:00:01,000 --> 00:00:02,500\nHello\n\n2\n00:00:03,000 --> 00:00:04,000\nWorld\n").format == .srt)216        #expect(detect("WEBVTT\n\n00:00:01.000 --> 00:00:02.500\nHello\n").format == .vtt)217        #expect(detect("[Script Info]\nScriptType: v4.00+\n\n[V4+ Styles]\nFormat: Name\n\n[Events]\n").format == .ass)218        #expect(detect("[Script Info]\nScriptType: v4.00\n\n[V4 Styles]\nFormat: Name\n").format == .ssa)219        #expect(detect("{0}{25}Hello\n{30}{60}World|second line\n").format == .sub)220        #expect(detect("1\nThis is just a numbered note.\n").format == .txt)221    }222223    @Test func textual3DFormats() {224        #expect(detect("# cube\nv 0 0 0\nv 1 0 0\nv 1 1 0\nvn 0 0 1\nf 1 2 3\n").format == .obj)225        #expect(detect("#usda 1.0\n(\n    defaultPrim = \"World\"\n)\n").format == .usda)226        #expect(detect("solid cube\n  facet normal 0 0 1\n    outer loop\n      vertex 0 0 0\n    endloop\n  endfacet\nendsolid cube\n").format == .stl)227        #expect(detect("ply\nformat ascii 1.0\nelement vertex 0\nend_header\n").format == .ply)228    }229230    @Test func plainText() {231        let result = detect("Just a few words without any structure.\nAnother line follows here.\n")232        #expect(result.format == .txt && result.confidence == .likely)233        #expect(detect("   \n\n").format == .txt)234    }235}236237@Suite struct DetectorRefinementTests {238    @Test func looseTextFollowsTheExtension() {239        let md = detect("Just a few words.\n", "README.md")240        #expect(md.format == .markdown && !md.mismatch)241        let csv = detect("single line, with one comma\n", "one.csv")242        #expect(csv.format == .csv && !csv.mismatch)243        let tex = detect("Some included fragment without commands.\n", "part.tex")244        #expect(tex.format == .latex && !tex.mismatch)245        // Markdown saved as .txt stays plain text for the user.246        #expect(detect("# Title\n\n**bold** and [a](b)\n", "notes.txt").format == .txt)247    }248249    @Test func structuredNamesAreNotSatisfiedByProse() {250        #expect(detect("hello there\n", "data.json").mismatch)251        #expect(detect("hello there\n", "movie.srt").mismatch)252        #expect(detect("hello there\n", "page.html").mismatch)253    }254255    @Test func structuredContentUnderATextNameIsKept() {256        let result = detect("{\"a\": 1}", "dump.txt")257        #expect(result.format == .json && !result.mismatch)258    }259260    @Test func typeScriptIsNotATransportStream() {261        let result = detect("export const answer: number = 42;\nconsole.log(answer);\n", "index.ts")262        #expect(result.format == .txt && result.mismatch)263    }264265    @Test func unknownExtensionsNeverMismatch() {266        #expect(!detect("{\"a\": 1}", "file.weird").mismatch)267        #expect(detect("{\"a\": 1}", "file.weird").format == .json)268    }269}270271/// Deterministic generator so failures are reproducible.272private struct SplitMix64: RandomNumberGenerator {273    var state: UInt64274    mutating func next() -> UInt64 {275        state &+= 0x9E37_79B9_7F4A_7C15276        var z = state277        z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9278        z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB279        return z ^ (z >> 31)280    }281}282283@Suite struct DetectorFuzzTests {284    static let corpus: [[UInt8]] = {285        let types = ascii("<?xml version=\"1.0\"?><Types/>")286        return [287            Fixtures.png, Fixtures.apng, Fixtures.mp3Frames, Fixtures.tar(), Fixtures.gzip(Fixtures.tar()),288            compress(Fixtures.tar(), COMPRESSION_LZMA),289            Fixtures.zip([("[Content_Types].xml", types), ("word/document.xml", ascii("<w/>"))], deflated: ["[Content_Types].xml"]),290            Fixtures.zip([("mimetype", ascii("application/epub+zip")), ("content.opf", ascii("<x/>"))]),291            Fixtures.ole2(streams: ["WordDocument"]),292            Fixtures.ftyp("M4A ", compatible: ["isom"]) + be32(60) + ascii("moov") + be32(52) + ascii("stsd") + be32(0) + be32(1) + be32(36) + ascii("alac") + zeros(28),293            [0x49, 0x49, 0x2A, 0x00] + le32(8) + le16(2) + le16(256) + le16(3) + le32(1) + le32(1) + le16(0xC612) + le16(1) + le32(4) + le32(1) + le32(0),294            ascii("OggS") + zeros(22) + [1, 19] + ascii("OpusHead") + zeros(11),295            [0x1A, 0x45, 0xDF, 0xA3, 0x9F, 0x42, 0x86, 0x81, 0x01, 0x42, 0x82, 0x84] + ascii("webm"),296            ascii("ID3") + [3, 0, 0, 0, 0, 0, 10] + zeros(10) + Fixtures.mp3Frames,297            ascii("SQLite format 3\0") + zeros(84),298            ascii("P6\n# c\n2 2\n255\n") + zeros(12),299            zeros(80) + le32(2) + zeros(100),300            ascii("{\"type\": \"FeatureCollection\", \"features\": [{\"type\": \"Feature\"}]}"),301            ascii("<?xml version=\"1.0\"?>\n<!DOCTYPE svg [ <!ENTITY a \"b\"> ]>\n<svg xmlns=\"http://www.w3.org/2000/svg\"/>"),302            ascii("1\n00:00:01,000 --> 00:00:02,500\nHello\n"),303            ascii("---\nname: x\nlist:\n  - a\n"),304            ascii("[package]\nname = \"demo\"\n"),305            ascii("id,name\n1,\"a, b\"\n2,\"c\nd\"\n"),306            ascii("Title\n=====\n\n.. note::\n   x\n"),307            Array("héllo wörld\n".data(using: .utf16LittleEndian)!),308            [0xEF, 0xBB, 0xBF] + ascii("# Title\n\n**b** [a](b)\n"),309        ]310    }()311312    @Test func randomBytesNeverCrash() {313        var rng = SplitMix64(state: 0x0E1_C0DE)314        for round in 0..<3000 {315            let length = round % 7 == 0 ? Int.random(in: 0...70_000, using: &rng) : Int.random(in: 0...600, using: &rng)316            var bytes = [UInt8](repeating: 0, count: length)317            for i in bytes.indices { bytes[i] = UInt8.random(in: 0...255, using: &rng) }318            let result = FileTypeDetector.detect(data: Data(bytes), fileName: round % 3 == 0 ? "x.zip" : nil)319            #expect(result.format != nil || result.confidence == .extensionOnly)320        }321    }322323    @Test func randomBytesBehindEveryKnownMagicNeverCrash() {324        var rng = SplitMix64(state: 42)325        let magics: [[UInt8]] = [326            [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A], [0xFF, 0xD8, 0xFF], ascii("GIF89a"), [0x49, 0x49, 0x2A, 0x00], [0x4D, 0x4D, 0x00, 0x2A],327            ascii("BM"), ascii("RIFF"), [0, 0, 0, 24] + ascii("ftyp"), [0, 0, 0, 0] + ascii("ftyp"), [0xFF, 0xFF, 0xFF, 0xFF] + ascii("ftypM4A "),328            [0, 0, 0, 1] + ascii("ftypM4A "), [0x1A, 0x45, 0xDF, 0xA3], ascii("OggS"), ascii("ID3"), [0xFF, 0xFB], [0xFF, 0xF1], ascii("FORM"),329            [0x30, 0x26, 0xB2, 0x75, 0x8E, 0x66, 0xCF, 0x11, 0xA6, 0xD9, 0x00, 0xAA, 0x00, 0x62, 0xCE, 0x6C], [0x47], ascii("8BPS"), [0, 0, 1, 0],330            ascii("icns"), ascii("P6 "), ascii("%PDF-"), [0x50, 0x4B, 0x03, 0x04], [0x50, 0x4B, 0x05, 0x06], [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1],331            [0x1F, 0x8B, 0x08], [0x1F, 0x8B, 0x08, 0x1C], ascii("BZh9"), [0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00], ascii("SQLite format 3\0"), ascii("PAR1"),332            [0x89, 0x48, 0x44, 0x46, 0x0D, 0x0A, 0x1A, 0x0A], [0, 1, 0, 0], ascii("glTF"), ascii("ply\n"), ascii("solid "), ascii("<?xml"), ascii("<"),333            ascii("{"), ascii("["), ascii("---\n"), [0xFF, 0xFE], [0xFE, 0xFF], [0xEF, 0xBB, 0xBF], ascii("{\\rtf"), ascii("1\n"), ascii("WEBVTT"),334        ]335        for magic in magics {336            for _ in 0..<60 {337                let length = Int.random(in: 0...2048, using: &rng)338                var bytes = magic339                for _ in 0..<length { bytes.append(UInt8.random(in: 0...255, using: &rng)) }340                _ = FileTypeDetector.detect(data: Data(bytes), fileName: "f.bin")341            }342        }343    }344345    @Test func everyTruncationOfEveryFixtureIsHandled() {346        for fixture in Self.corpus {347            let step = max(1, fixture.count / 700)348            for length in stride(from: 0, through: fixture.count, by: step) {349                _ = FileTypeDetector.detect(data: Data(fixture[0..<length]), fileName: nil)350            }351        }352    }353354    @Test func mutatedFixturesAreHandled() {355        var rng = SplitMix64(state: 7)356        for fixture in Self.corpus where !fixture.isEmpty {357            for _ in 0..<150 {358                var bytes = fixture359                for _ in 0..<Int.random(in: 1...6, using: &rng) {360                    let index = Int.random(in: 0..<bytes.count, using: &rng)361                    bytes[index] = [0x00, 0xFF, 0x7F, 0x80, UInt8.random(in: 0...255, using: &rng)].randomElement(using: &rng) ?? 0362                }363                _ = FileTypeDetector.detect(data: Data(bytes), fileName: "m.dat")364            }365        }366    }367368    @Test func hostileZipDirectoriesAreBounded() {369        // End record pointing far outside the file, with an absurd entry count.370        let bogus = [0x50, 0x4B, 0x03, 0x04] + zeros(26) + [0x50, 0x4B, 0x05, 0x06] + le16(0) + le16(0) + le16(0xFFFF) + le16(0xFFFF) + le32(0xFFFF_FFF0) + le32(0x7FFF_FFFF) + le16(0) as [UInt8]371        #expect(detect(bogus).format == .zip)372        // ZIP64 locator pointing nowhere.373        let zip64 = [0x50, 0x4B, 0x03, 0x04] + zeros(26) + [0x50, 0x4B, 0x06, 0x07] + le32(0) + le32(0xFFFF_FFFF) + le32(0x7FFF_FFFF) + le32(1)374            + [0x50, 0x4B, 0x05, 0x06] + le16(0) + le16(0) + le16(0xFFFF) + le16(0xFFFF) + le32(0xFFFF_FFFF) + le32(0xFFFF_FFFF) + le16(0) as [UInt8]375        #expect(detect(zip64).format == .zip)376        // Local header declaring a name longer than the file.377        let longName = [0x50, 0x4B, 0x03, 0x04] + zeros(22) + le16(0xFFFF) + le16(0xFFFF) + ascii("abc") as [UInt8]378        #expect(detect(longName).format == .zip)379        // Central directory entry with lengths running past the end.380        var cut = Fixtures.zip([("a.txt", ascii("a"))])381        if let cd = cut.find([0x50, 0x4B, 0x01, 0x02]) { cut[cd + 28] = 0xFF; cut[cd + 29] = 0xFF }382        #expect(detect(cut).format == .zip)383    }384385    @Test func hostileOLE2ChainsTerminate() {386        var looping = Fixtures.ole2(streams: ["WordDocument"])387        // FAT entry of the directory sector points to itself.388        looping.replaceSubrange((512 + 4)..<(512 + 8), with: le32(1))389        #expect(detect(looping).format == .doc)390        var absurd = Fixtures.ole2(streams: ["Workbook"])391        absurd.replaceSubrange(48..<52, with: le32(0x7FFF_FFF0))   // first directory sector far away392        #expect(detect(absurd).format == nil)393        var badShift = Fixtures.ole2(streams: ["Workbook"])394        badShift[30] = 31395        #expect(detect(badShift).format == nil)396    }397398    @Test func hostileBoxAndChunkLengths() {399        // PNG chunk length of 4 GiB.400        #expect(detect([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A] + be32(0xFFFF_FFFF) + ascii("IHDR") + zeros(8)).format == .png)401        // 64-bit box size close to Int64.max after the ftyp box.402        let huge = be32(16) + ascii("ftyp") + ascii("M4A ") + be32(0) + be32(1) + ascii("mdat") + [0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xF0] + zeros(8)403        #expect(detect(huge).format == .m4a)404        let negative = be32(16) + ascii("ftyp") + ascii("M4A ") + be32(0) + be32(1) + ascii("mdat") + [UInt8](repeating: 0xFF, count: 8)405        #expect(detect(negative).format == .m4a)406        // TIFF whose IFD offset points past the end.407        #expect(detect([0x49, 0x49, 0x2A, 0x00] + le32(0x7FFF_FFFF) + zeros(8)).format == .tiff)408        // ID3 tag claiming 256 MiB.409        #expect(detect(ascii("ID3") + [4, 0, 0, 0x7F, 0x7F, 0x7F, 0x7F] + zeros(64)).format == .mp3)410        // gzip header whose extra field runs past the end.411        #expect(detect([0x1F, 0x8B, 0x08, 0x04, 0, 0, 0, 0, 0, 3, 0xFF, 0xFF, 1, 2, 3]).format == .gz)412        // EBML DocType with a length beyond the data.413        #expect(detect([0x1A, 0x45, 0xDF, 0xA3, 0x42, 0x82, 0xFF, 0x77]).format == .mkv)414    }415416    @Test func deeplyNestedAndUnterminatedTextIsHandled() {417        #expect(detect(String(repeating: "[", count: 60_000)).format != nil)418        #expect(detect("{\"a\": \"" + String(repeating: "x", count: 100_000)).format != nil)419        #expect(detect("<?xml version=\"1.0\"" + String(repeating: " ", count: 70_000)).format != nil)420        #expect(detect("<!--" + String(repeating: "-", count: 70_000)).format != nil)421        #expect(detect(String(repeating: "\"", count: 9_999) + "\n" + String(repeating: ",", count: 5_000)).format != nil)422        #expect(detect(String(repeating: "- ", count: 30_000)).format != nil)423    }424}425
Files main
Menu

Theme