Repository Public License
1// ORIX Convert — Packages/EngineSubtitle/Tests/EngineSubtitleTests/ParserTests.swift2// Description: Real-world mess the parsers must tolerate, clean failures, and fuzz-style inputs that must never crash.3// Author: Simon-Pierre Boucher — contact@spboucher.ai4import ConvertCore5@testable import EngineSubtitle6import Foundation7import Testing89@Suite struct ParserTests {10 static func parse(_ text: String, _ format: FormatID) throws -> SubtitleDocument { try SubtitleParser.parse(data: Data(text.utf8), format: format) }1112 @Test func subRipToleratesMissingIndicesDecimalVariantsAndStrayBlankLines() throws {13 let messy = "\u{FEFF}\r\n\r\n00:00:01.000 --> 00:00:02.5\r\nNo index, dot decimals\r\n\r\n\r\n\r\n7\r\n0:00:03,000-->0:00:04,000 X1:100 X2:200\r\nNo spaces around the arrow\r\n\r\n" +14 "8\r\n00:00:05,000 --> 00:00:06,000\r\nFirst\r\n\r\nstill the same cue\r\n\r\n9\r\n00:00:07,000 --> 00:00:08,000\r\nGlued\r\n10\r\n00:00:09,000 --> 00:00:10,000\r\n42\r\n\r\n11\r\n00:00:11,000 --> 00:00:12,000\r\n"15 let doc = try Self.parse(messy, .srt)16 #expect(doc.cues.map { [$0.startMs, $0.endMs] } == [[1000, 2500], [3000, 4000], [5000, 6000], [7000, 8000], [9000, 10_000], [11_000, 12_000]])17 #expect(doc.cues.map(\.plainText) == ["No index, dot decimals", "No spaces around the arrow", "First\nstill the same cue", "Glued", "42", ""])18 #expect(doc.encoding.hadBOM && doc.encoding.evidence == .byteOrderMark)19 // Lone CR line endings.20 let cr = try Self.parse("1\r00:00:01,000 --> 00:00:02,000\rOld Mac\r\r2\r00:00:03,000 --> 00:00:04,000\rfile\r", .srt)21 #expect(cr.cues.map(\.plainText) == ["Old Mac", "file"])22 }2324 @Test func subRipReadsOverlappingAndUnknownTags() throws {25 let doc = try Self.parse("1\n00:00:01,000 --> 00:00:02,000\n<b>bold <i>both</b> italic</i> <u>u</u> <x>kept</x> 1 < 2 <font color=red face='Courier New'>r</font>{\\an3}{i}curly{/i}\n", .srt)26 let cue = doc.cues[0]27 #expect(cue.plainText == "bold both italic u <x>kept</x> 1 < 2 rcurly")28 #expect(cue.spans[0].attributes.bold && !cue.spans[0].attributes.italic)29 #expect(cue.spans[1].attributes.bold && cue.spans[1].attributes.italic)30 #expect(!cue.spans[2].attributes.bold)31 #expect(cue.spans.contains { $0.attributes.color == 0xFF0000 && $0.attributes.font == "Courier New" })32 #expect(cue.spans.last?.attributes.italic == true && cue.alignment == 3)33 }3435 @Test func webVTTReadsNotesStylesRegionsIdentifiersAndSettings() throws {36 let vtt = """37 WEBVTT - Title text38 Kind: captions39 Language: en4041 NOTE42 multi-line43 note4445 STYLE46 ::cue(b) { color: peachpuff; }4748 REGION49 id:fred50 width:40%5152 153 00:01.000 --> 00:02.000 region:fred54 <v.loud Fred>Hi & <bye> now</v>5556 named cue57 00:00:03.000 --> 00:00:04.000 line:0 position:10% align:start size:35%58 <c.red.bg_blue>red</c> <c.custom>custom</c> <00:00:03.500>timed <ruby>base<rt>text</rt></ruby>5960 00:00:05.000 --> 00:00:06.000 line:90% position:50%61 Both axes62 00:00:07.000 --> 00:00:08.00063 Missing blank line6465 this is not a cue6667 """68 let doc = try Self.parse(vtt, .vtt)69 #expect(doc.cues.count == 4)70 #expect(doc.report.vttNotes == 1 && doc.report.vttStyleBlocks == 1 && doc.report.vttRegions == 1 && doc.report.skippedBlocks == 1)71 #expect(doc.cues[0].plainText == "Hi & <bye>\u{00A0}now" && doc.cues[0].identifier == "1")72 #expect(doc.cues[0].features.contains(.annotations) && doc.cues[0].features.contains(.namedStyle))73 #expect(doc.cues[1].identifier == "named cue" && doc.cues[1].alignment == 7)74 #expect(doc.cues[1].spans[0].attributes.color == 0xFF0000 && doc.cues[1].features.contains(.karaoke))75 #expect(doc.cues[1].plainText == "red custom timed basetext")76 #expect(doc.cues[2].positionX == 50 && doc.cues[2].positionY == 90 && doc.cues[2].alignment == 2)77 #expect(doc.cues[3].plainText == "Missing blank line" && doc.cues[3].startMs == 7000)78 // VTT → VTT keeps the header, the STYLE / REGION blocks, identifiers, settings and raw markup.79 let out = String(decoding: SubtitlePipeline.run(document: doc, to: .vtt, settings: .init()).data, as: UTF8.self)80 #expect(out.hasPrefix("WEBVTT - Title text\n\nSTYLE\n::cue(b) { color: peachpuff; }\n\nREGION\nid:fred\nwidth:40%\n\n1\n00:00:01.000 --> 00:00:02.000 region:fred\n<v.loud Fred>Hi & <bye> now</v>\n\n"))81 #expect(out.contains("named cue\n00:00:03.000 --> 00:00:04.000 line:0 position:10% align:start size:35%\n"))82 }8384 @Test func assHonoursReorderedFormatLinesAndCommasInText() throws {85 let ass = """86 [Script Info]87 ; a comment88 ScriptType: v4.00+89 PlayResX: 64090 PlayResY: 3609192 [V4+ Styles]93 Format: Fontsize, Name, Fontname, Alignment94 Style: 20,Alt,Courier New,89596 [Events]97 Format: Start, End, Layer, Effect, Style, Text98 Dialogue: 0:00:01.00,0:00:02.50,3,,Alt,Hello, world, with, commas99 Dialogue: 0:00:03.00,0:00:04.00,0,Scroll up;10;20,Alt,{\\pos(320,90)\\fad(100,100)}Positioned{\\r}, {\\k20}ka{\\k30}ra100 Dialogue: 0:00:05.00,0:00:06.00,0,,Alt,{\\p1}m 0 0 l 10 0 10 10 0 10{\\p0}101 Dialogue: 0:00:07.00,0:00:08.00,0,,Alt,{\\fnImpact\\1c&HFF0000&\\b700\\u1}styled{\\fn\\c\\b0} plain\\Nnext\\hline {not a tag}102 Dialogue: garbage103 Picture: 0:00:01.00,0:00:02.00,0,,Alt,logo.bmp104105 [Fonts]106 fontname: custom_0.ttf107 M3I4N2E@&Y108109 [Graphics]110 filename: logo.bmp111 M3I4N2E@&Y112 """113 let doc = try Self.parse(ass, .ass)114 #expect(doc.cues.count == 4 && doc.report.skippedBlocks == 1)115 #expect(doc.cues[0].plainText == "Hello, world, with, commas" && doc.cues[0].startMs == 1000 && doc.cues[0].endMs == 2500)116 #expect(doc.cues[0].raw?.layer == "3" && doc.cues[0].raw?.style == "Alt")117 #expect(doc.cues[1].positionX == 50 && doc.cues[1].positionY == 25)118 #expect(doc.cues[1].features.isSuperset(of: [.effects, .karaoke, .namedStyle]) && doc.cues[1].plainText == "Positioned, kara")119 #expect(doc.cues[2].features.contains(.drawing) && doc.cues[2].plainText.isEmpty)120 let styled = doc.cues[3]121 #expect(styled.plainText == "styled plain\nnext\u{00A0}line ")122 #expect(styled.spans[0].attributes.font == "Impact" && styled.spans[0].attributes.color == 0x0000FF && styled.spans[0].attributes.bold && styled.spans[0].attributes.underline)123 #expect(styled.spans[1].attributes.font == nil && styled.spans[1].attributes.color == nil && !styled.spans[1].attributes.bold && styled.spans[1].attributes.underline)124 #expect(doc.styleCount == 1 && doc.embeddedFontCount == 1 && doc.embeddedGraphicCount == 1 && doc.ass?.otherEvents == 1)125 #expect(doc.ass?.styles[0].fields["fontname"] == "Courier New")126 // The drawing is dropped (not written as garbage text) when leaving ASS, and everything else is counted.127 let result = SubtitlePipeline.run(document: doc, to: .srt, settings: .init())128 let text = String(decoding: result.data, as: UTF8.self)129 #expect(result.cues.count == 3 && !text.contains("m 0 0 l"))130 let codes = Set(result.warnings.map(\.code))131 #expect(codes.isSuperset(of: ["drawings-dropped", "position-dropped", "karaoke-dropped", "effects-dropped", "named-styles-dropped",132 "style-definitions-dropped", "embedded-files-dropped", "events-dropped", "blocks-skipped"]))133 #expect(result.warnings.contains { $0.code == "embedded-files-dropped" && $0.message.hasPrefix("1 embedded font and 1 embedded picture") })134 // ASS → ASS keeps raw override tags, the layer, the sections and the drawing.135 let same = String(decoding: SubtitlePipeline.run(document: doc, to: .ass, settings: .init()).data, as: UTF8.self)136 #expect(same.contains("Dialogue: 3,0:00:01.00,0:00:02.50,Alt,,0,0,0,,Hello, world, with, commas\n"))137 #expect(same.contains(",Scroll up;10;20,{\\pos(320,90)\\fad(100,100)}Positioned{\\r}, {\\k20}ka{\\k30}ra\n"))138 #expect(same.contains("{\\p1}m 0 0 l 10 0 10 10 0 10{\\p0}") && same.contains("[Fonts]\nfontname: custom_0.ttf\nM3I4N2E@&Y") && same.contains("; a comment"))139 #expect(same.contains("Style: Alt,Courier New,20,&H00FFFFFF,"))140 }141142 @Test func microDVDReadsControlCodes() throws {143 let doc = try Self.parse("{10}{20}{Y:i}all italic|second line\n{30}{40}{y:b}{c:$0000FF}red bold|plain line\n{50}{60}/slash italic|{f:Courier}{s:20}mono\nnot a cue\n[70][80]square\n{90}{100}A|B|C\n", .sub)144 #expect(doc.cues.count == 4 && doc.report.skippedBlocks == 2)145 #expect(doc.cues[0].spans.allSatisfy { $0.attributes.italic } && doc.cues[0].plainText == "all italic\nsecond line")146 #expect(doc.cues[1].spans[0].attributes.bold && doc.cues[1].spans[0].attributes.color == 0xFF0000)147 #expect(doc.cues[1].spans.last?.attributes.bold == false)148 #expect(doc.cues[2].spans[0].attributes.italic && doc.cues[2].spans.last?.attributes.font == "Courier")149 #expect(doc.cues[3].plainText == "A\nB\nC" && doc.cues[3].frames == 90...100)150 }151152 @Test(arguments: SubtitleEngine.formats)153 func emptyAndCueLessFilesFailCleanly(format: FormatID) async throws {154 let scratch = try Scratch(); defer { scratch.cleanUp() }155 for (n, content) in ["", "\n\n\n", "just some prose\nwithout any cue\n", "WEBVTT\n\nNOTE nothing else\n", "[Script Info]\nTitle: empty\n\n[Events]\nFormat: Start, End, Text\n"].enumerated() {156 let url = try scratch.file("e\(n)." + Fixtures.ext(format), content)157 await #expect(throws: ConversionError.self) { _ = try await SubtitleEngine().inspect(url, format: format) }158 await #expect(throws: ConversionError.self) {159 _ = try await SubtitleEngine().convert(.init(source: url, sourceFormat: format, destinationFormat: format == .srt ? .vtt : .srt),160 context: .init(workDirectory: try scratch.work()))161 }162 }163 let missing = scratch.root.appendingPathComponent("missing.srt")164 await #expect(throws: ConversionError.self) { _ = try await SubtitleEngine().inspect(missing, format: format) }165 #expect(SubtitleEngine().preflight(.init(source: missing, sourceFormat: format, destinationFormat: format == .srt ? .vtt : .srt), inspection: nil).first?.code == "preflight-failed")166 }167168 @Test func oversizedFilesAreRefused() throws {169 let big = Data(count: SubtitleParser.maximumFileSize + 1)170 #expect(throws: ConversionError.limitExceeded("Subtitle files larger than 64 MB are not accepted.")) { _ = try SubtitleParser.parse(data: big, format: .srt) }171 }172173 @Test(arguments: SubtitleEngine.formats)174 func garbageNeverCrashesTheParsers(format: FormatID) throws {175 var rng = TimingTests.LCG(state: UInt64(abs(format.rawValue.hashValue % 1000)) + 7)176 let alphabet = Array("0123456789:,.-> {}[]<>/\\|\n\r\tabcXYZ&;%$éあ\u{0}\u{FEFF}WEBVTTDialogue:Format:Style:NOTEpos(k1an8b1c&H".unicodeScalars)177 var parsed = 0178 for round in 0..<400 {179 var data = Data()180 if round % 4 == 0 {181 // Pure random bytes.182 for _ in 0..<(1 + rng.next(600)) { data.append(UInt8(rng.next(256))) }183 } else if round % 4 == 1 {184 // Random characters from the syntax alphabet.185 var s = String.UnicodeScalarView()186 for _ in 0..<(1 + rng.next(800)) { s.append(alphabet[rng.next(alphabet.count)]) }187 data = Data(String(s).utf8)188 } else {189 // A valid fixture with random bytes overwritten, deleted or truncated.190 var bytes = [UInt8](Fixtures.text(format).utf8)191 for _ in 0..<(1 + rng.next(12)) { bytes[rng.next(bytes.count)] = UInt8(rng.next(256)) }192 if round % 4 == 3 { bytes.removeLast(rng.next(bytes.count)) }193 data = Data(bytes)194 }195 // Must return or throw a ConversionError; then every writer must cope with whatever was parsed.196 guard let doc = try? SubtitleParser.parse(data: data, format: format) else { continue }197 parsed += 1198 for destination in SubtitleEngine.formats {199 var settings = SubtitleSettings()200 settings.stripHearingImpaired = round % 2 == 0; settings.collapseWhitespace = true; settings.fixOCR = true; settings.wrapLength = 20201 settings.sort = true; settings.fixOverlaps = true; settings.fixDurations = true; settings.shiftMs = -500202 let result = SubtitlePipeline.run(document: doc, to: destination, settings: settings)203 if !result.cues.isEmpty { _ = try? SubtitleParser.parse(data: result.data, format: destination) }204 }205 }206 #expect(parsed > 50) // the mutated fixtures mostly still parse: the loop really exercised the writers207 }208209 @Test func absurdTimestampsAndHugeLinesAreBounded() throws {210 let long = String(repeating: "x", count: 200_000)211 let doc = try Self.parse("1\n00:00:01,000 --> 00:00:02,000\n\(long)\n\n2\n99999999999999999999:00:00,000 --> 00:00:03,000\nbad\n\n3\n00:99:99,000 --> 01:00:00,000\nodd\n", .srt)212 #expect(doc.cues.count == 2 && doc.report.skippedBlocks == 1)213 #expect(doc.cues[0].plainText.count == 200_000, "\(doc.cues[0].plainText.count) \(doc.cues.map(\.startMs))")214 #expect(doc.cues[1].startMs == (99 * 60 + 99) * 1000)215 #expect(SubtitleTime.parse("1:2:3:4") == nil && SubtitleTime.parse("") == nil && SubtitleTime.parse("12") == nil && SubtitleTime.parse("1.5") == 1500)216 }217}218