SPB Git

Repository Public License

ParserTests.swift 13.7 KB 218 lines swift

1// ORIX Convert — Packages/EngineSubtitle/Tests/EngineSubtitleTests/ParserTests.swift2// Description: Real-world mess the parsers must tolerate, clean failures, and fuzz-style inputs that must never crash.3// Author: Simon-Pierre Boucher — contact@spboucher.ai4import ConvertCore5@testable import EngineSubtitle6import Foundation7import Testing89@Suite struct ParserTests {10    static func parse(_ text: String, _ format: FormatID) throws -> SubtitleDocument { try SubtitleParser.parse(data: Data(text.utf8), format: format) }1112    @Test func subRipToleratesMissingIndicesDecimalVariantsAndStrayBlankLines() throws {13        let messy = "\u{FEFF}\r\n\r\n00:00:01.000 --> 00:00:02.5\r\nNo index, dot decimals\r\n\r\n\r\n\r\n7\r\n0:00:03,000-->0:00:04,000  X1:100 X2:200\r\nNo spaces around the arrow\r\n\r\n" +14            "8\r\n00:00:05,000 --> 00:00:06,000\r\nFirst\r\n\r\nstill the same cue\r\n\r\n9\r\n00:00:07,000 --> 00:00:08,000\r\nGlued\r\n10\r\n00:00:09,000 --> 00:00:10,000\r\n42\r\n\r\n11\r\n00:00:11,000 --> 00:00:12,000\r\n"15        let doc = try Self.parse(messy, .srt)16        #expect(doc.cues.map { [$0.startMs, $0.endMs] } == [[1000, 2500], [3000, 4000], [5000, 6000], [7000, 8000], [9000, 10_000], [11_000, 12_000]])17        #expect(doc.cues.map(\.plainText) == ["No index, dot decimals", "No spaces around the arrow", "First\nstill the same cue", "Glued", "42", ""])18        #expect(doc.encoding.hadBOM && doc.encoding.evidence == .byteOrderMark)19        // Lone CR line endings.20        let cr = try Self.parse("1\r00:00:01,000 --> 00:00:02,000\rOld Mac\r\r2\r00:00:03,000 --> 00:00:04,000\rfile\r", .srt)21        #expect(cr.cues.map(\.plainText) == ["Old Mac", "file"])22    }2324    @Test func subRipReadsOverlappingAndUnknownTags() throws {25        let doc = try Self.parse("1\n00:00:01,000 --> 00:00:02,000\n<b>bold <i>both</b> italic</i> <u>u</u> <x>kept</x> 1 < 2 <font color=red face='Courier New'>r</font>{\\an3}{i}curly{/i}\n", .srt)26        let cue = doc.cues[0]27        #expect(cue.plainText == "bold both italic u <x>kept</x> 1 < 2 rcurly")28        #expect(cue.spans[0].attributes.bold && !cue.spans[0].attributes.italic)29        #expect(cue.spans[1].attributes.bold && cue.spans[1].attributes.italic)30        #expect(!cue.spans[2].attributes.bold)31        #expect(cue.spans.contains { $0.attributes.color == 0xFF0000 && $0.attributes.font == "Courier New" })32        #expect(cue.spans.last?.attributes.italic == true && cue.alignment == 3)33    }3435    @Test func webVTTReadsNotesStylesRegionsIdentifiersAndSettings() throws {36        let vtt = """37        WEBVTT - Title text38        Kind: captions39        Language: en4041        NOTE42        multi-line43        note4445        STYLE46        ::cue(b) { color: peachpuff; }4748        REGION49        id:fred50        width:40%5152        153        00:01.000 --> 00:02.000 region:fred54        <v.loud Fred>Hi &amp; &lt;bye&gt;&nbsp;now</v>5556        named cue57        00:00:03.000 --> 00:00:04.000 line:0 position:10% align:start size:35%58        <c.red.bg_blue>red</c> <c.custom>custom</c> <00:00:03.500>timed <ruby>base<rt>text</rt></ruby>5960        00:00:05.000 --> 00:00:06.000 line:90% position:50%61        Both axes62        00:00:07.000 --> 00:00:08.00063        Missing blank line6465        this is not a cue6667        """68        let doc = try Self.parse(vtt, .vtt)69        #expect(doc.cues.count == 4)70        #expect(doc.report.vttNotes == 1 && doc.report.vttStyleBlocks == 1 && doc.report.vttRegions == 1 && doc.report.skippedBlocks == 1)71        #expect(doc.cues[0].plainText == "Hi & <bye>\u{00A0}now" && doc.cues[0].identifier == "1")72        #expect(doc.cues[0].features.contains(.annotations) && doc.cues[0].features.contains(.namedStyle))73        #expect(doc.cues[1].identifier == "named cue" && doc.cues[1].alignment == 7)74        #expect(doc.cues[1].spans[0].attributes.color == 0xFF0000 && doc.cues[1].features.contains(.karaoke))75        #expect(doc.cues[1].plainText == "red custom timed basetext")76        #expect(doc.cues[2].positionX == 50 && doc.cues[2].positionY == 90 && doc.cues[2].alignment == 2)77        #expect(doc.cues[3].plainText == "Missing blank line" && doc.cues[3].startMs == 7000)78        // VTT → VTT keeps the header, the STYLE / REGION blocks, identifiers, settings and raw markup.79        let out = String(decoding: SubtitlePipeline.run(document: doc, to: .vtt, settings: .init()).data, as: UTF8.self)80        #expect(out.hasPrefix("WEBVTT - Title text\n\nSTYLE\n::cue(b) { color: peachpuff; }\n\nREGION\nid:fred\nwidth:40%\n\n1\n00:00:01.000 --> 00:00:02.000 region:fred\n<v.loud Fred>Hi &amp; &lt;bye&gt;&nbsp;now</v>\n\n"))81        #expect(out.contains("named cue\n00:00:03.000 --> 00:00:04.000 line:0 position:10% align:start size:35%\n"))82    }8384    @Test func assHonoursReorderedFormatLinesAndCommasInText() throws {85        let ass = """86        [Script Info]87        ; a comment88        ScriptType: v4.00+89        PlayResX: 64090        PlayResY: 3609192        [V4+ Styles]93        Format: Fontsize, Name, Fontname, Alignment94        Style: 20,Alt,Courier New,89596        [Events]97        Format: Start, End, Layer, Effect, Style, Text98        Dialogue: 0:00:01.00,0:00:02.50,3,,Alt,Hello, world, with, commas99        Dialogue: 0:00:03.00,0:00:04.00,0,Scroll up;10;20,Alt,{\\pos(320,90)\\fad(100,100)}Positioned{\\r}, {\\k20}ka{\\k30}ra100        Dialogue: 0:00:05.00,0:00:06.00,0,,Alt,{\\p1}m 0 0 l 10 0 10 10 0 10{\\p0}101        Dialogue: 0:00:07.00,0:00:08.00,0,,Alt,{\\fnImpact\\1c&HFF0000&\\b700\\u1}styled{\\fn\\c\\b0} plain\\Nnext\\hline {not a tag}102        Dialogue: garbage103        Picture: 0:00:01.00,0:00:02.00,0,,Alt,logo.bmp104105        [Fonts]106        fontname: custom_0.ttf107        M3I4N2E@&Y108109        [Graphics]110        filename: logo.bmp111        M3I4N2E@&Y112        """113        let doc = try Self.parse(ass, .ass)114        #expect(doc.cues.count == 4 && doc.report.skippedBlocks == 1)115        #expect(doc.cues[0].plainText == "Hello, world, with, commas" && doc.cues[0].startMs == 1000 && doc.cues[0].endMs == 2500)116        #expect(doc.cues[0].raw?.layer == "3" && doc.cues[0].raw?.style == "Alt")117        #expect(doc.cues[1].positionX == 50 && doc.cues[1].positionY == 25)118        #expect(doc.cues[1].features.isSuperset(of: [.effects, .karaoke, .namedStyle]) && doc.cues[1].plainText == "Positioned, kara")119        #expect(doc.cues[2].features.contains(.drawing) && doc.cues[2].plainText.isEmpty)120        let styled = doc.cues[3]121        #expect(styled.plainText == "styled plain\nnext\u{00A0}line ")122        #expect(styled.spans[0].attributes.font == "Impact" && styled.spans[0].attributes.color == 0x0000FF && styled.spans[0].attributes.bold && styled.spans[0].attributes.underline)123        #expect(styled.spans[1].attributes.font == nil && styled.spans[1].attributes.color == nil && !styled.spans[1].attributes.bold && styled.spans[1].attributes.underline)124        #expect(doc.styleCount == 1 && doc.embeddedFontCount == 1 && doc.embeddedGraphicCount == 1 && doc.ass?.otherEvents == 1)125        #expect(doc.ass?.styles[0].fields["fontname"] == "Courier New")126        // The drawing is dropped (not written as garbage text) when leaving ASS, and everything else is counted.127        let result = SubtitlePipeline.run(document: doc, to: .srt, settings: .init())128        let text = String(decoding: result.data, as: UTF8.self)129        #expect(result.cues.count == 3 && !text.contains("m 0 0 l"))130        let codes = Set(result.warnings.map(\.code))131        #expect(codes.isSuperset(of: ["drawings-dropped", "position-dropped", "karaoke-dropped", "effects-dropped", "named-styles-dropped",132                                      "style-definitions-dropped", "embedded-files-dropped", "events-dropped", "blocks-skipped"]))133        #expect(result.warnings.contains { $0.code == "embedded-files-dropped" && $0.message.hasPrefix("1 embedded font and 1 embedded picture") })134        // ASS → ASS keeps raw override tags, the layer, the sections and the drawing.135        let same = String(decoding: SubtitlePipeline.run(document: doc, to: .ass, settings: .init()).data, as: UTF8.self)136        #expect(same.contains("Dialogue: 3,0:00:01.00,0:00:02.50,Alt,,0,0,0,,Hello, world, with, commas\n"))137        #expect(same.contains(",Scroll up;10;20,{\\pos(320,90)\\fad(100,100)}Positioned{\\r}, {\\k20}ka{\\k30}ra\n"))138        #expect(same.contains("{\\p1}m 0 0 l 10 0 10 10 0 10{\\p0}") && same.contains("[Fonts]\nfontname: custom_0.ttf\nM3I4N2E@&Y") && same.contains("; a comment"))139        #expect(same.contains("Style: Alt,Courier New,20,&H00FFFFFF,"))140    }141142    @Test func microDVDReadsControlCodes() throws {143        let doc = try Self.parse("{10}{20}{Y:i}all italic|second line\n{30}{40}{y:b}{c:$0000FF}red bold|plain line\n{50}{60}/slash italic|{f:Courier}{s:20}mono\nnot a cue\n[70][80]square\n{90}{100}A|B|C\n", .sub)144        #expect(doc.cues.count == 4 && doc.report.skippedBlocks == 2)145        #expect(doc.cues[0].spans.allSatisfy { $0.attributes.italic } && doc.cues[0].plainText == "all italic\nsecond line")146        #expect(doc.cues[1].spans[0].attributes.bold && doc.cues[1].spans[0].attributes.color == 0xFF0000)147        #expect(doc.cues[1].spans.last?.attributes.bold == false)148        #expect(doc.cues[2].spans[0].attributes.italic && doc.cues[2].spans.last?.attributes.font == "Courier")149        #expect(doc.cues[3].plainText == "A\nB\nC" && doc.cues[3].frames == 90...100)150    }151152    @Test(arguments: SubtitleEngine.formats)153    func emptyAndCueLessFilesFailCleanly(format: FormatID) async throws {154        let scratch = try Scratch(); defer { scratch.cleanUp() }155        for (n, content) in ["", "\n\n\n", "just some prose\nwithout any cue\n", "WEBVTT\n\nNOTE nothing else\n", "[Script Info]\nTitle: empty\n\n[Events]\nFormat: Start, End, Text\n"].enumerated() {156            let url = try scratch.file("e\(n)." + Fixtures.ext(format), content)157            await #expect(throws: ConversionError.self) { _ = try await SubtitleEngine().inspect(url, format: format) }158            await #expect(throws: ConversionError.self) {159                _ = try await SubtitleEngine().convert(.init(source: url, sourceFormat: format, destinationFormat: format == .srt ? .vtt : .srt),160                                                       context: .init(workDirectory: try scratch.work()))161            }162        }163        let missing = scratch.root.appendingPathComponent("missing.srt")164        await #expect(throws: ConversionError.self) { _ = try await SubtitleEngine().inspect(missing, format: format) }165        #expect(SubtitleEngine().preflight(.init(source: missing, sourceFormat: format, destinationFormat: format == .srt ? .vtt : .srt), inspection: nil).first?.code == "preflight-failed")166    }167168    @Test func oversizedFilesAreRefused() throws {169        let big = Data(count: SubtitleParser.maximumFileSize + 1)170        #expect(throws: ConversionError.limitExceeded("Subtitle files larger than 64 MB are not accepted.")) { _ = try SubtitleParser.parse(data: big, format: .srt) }171    }172173    @Test(arguments: SubtitleEngine.formats)174    func garbageNeverCrashesTheParsers(format: FormatID) throws {175        var rng = TimingTests.LCG(state: UInt64(abs(format.rawValue.hashValue % 1000)) + 7)176        let alphabet = Array("0123456789:,.-> {}[]<>/\\|\n\r\tabcXYZ&;%$éあ\u{0}\u{FEFF}WEBVTTDialogue:Format:Style:NOTEpos(k1an8b1c&H".unicodeScalars)177        var parsed = 0178        for round in 0..<400 {179            var data = Data()180            if round % 4 == 0 {181                // Pure random bytes.182                for _ in 0..<(1 + rng.next(600)) { data.append(UInt8(rng.next(256))) }183            } else if round % 4 == 1 {184                // Random characters from the syntax alphabet.185                var s = String.UnicodeScalarView()186                for _ in 0..<(1 + rng.next(800)) { s.append(alphabet[rng.next(alphabet.count)]) }187                data = Data(String(s).utf8)188            } else {189                // A valid fixture with random bytes overwritten, deleted or truncated.190                var bytes = [UInt8](Fixtures.text(format).utf8)191                for _ in 0..<(1 + rng.next(12)) { bytes[rng.next(bytes.count)] = UInt8(rng.next(256)) }192                if round % 4 == 3 { bytes.removeLast(rng.next(bytes.count)) }193                data = Data(bytes)194            }195            // Must return or throw a ConversionError; then every writer must cope with whatever was parsed.196            guard let doc = try? SubtitleParser.parse(data: data, format: format) else { continue }197            parsed += 1198            for destination in SubtitleEngine.formats {199                var settings = SubtitleSettings()200                settings.stripHearingImpaired = round % 2 == 0; settings.collapseWhitespace = true; settings.fixOCR = true; settings.wrapLength = 20201                settings.sort = true; settings.fixOverlaps = true; settings.fixDurations = true; settings.shiftMs = -500202                let result = SubtitlePipeline.run(document: doc, to: destination, settings: settings)203                if !result.cues.isEmpty { _ = try? SubtitleParser.parse(data: result.data, format: destination) }204            }205        }206        #expect(parsed > 50)        // the mutated fixtures mostly still parse: the loop really exercised the writers207    }208209    @Test func absurdTimestampsAndHugeLinesAreBounded() throws {210        let long = String(repeating: "x", count: 200_000)211        let doc = try Self.parse("1\n00:00:01,000 --> 00:00:02,000\n\(long)\n\n2\n99999999999999999999:00:00,000 --> 00:00:03,000\nbad\n\n3\n00:99:99,000 --> 01:00:00,000\nodd\n", .srt)212        #expect(doc.cues.count == 2 && doc.report.skippedBlocks == 1)213        #expect(doc.cues[0].plainText.count == 200_000, "\(doc.cues[0].plainText.count) \(doc.cues.map(\.startMs))")214        #expect(doc.cues[1].startMs == (99 * 60 + 99) * 1000)215        #expect(SubtitleTime.parse("1:2:3:4") == nil && SubtitleTime.parse("") == nil && SubtitleTime.parse("12") == nil && SubtitleTime.parse("1.5") == 1500)216    }217}218
Files main
Menu

Theme