Repository Public License
1// ORIX Convert — Packages/EngineSubtitle/Tests/EngineSubtitleTests/EncodingAndCleanupTests.swift2// Description: Encoding detection and output, text clean-up options, preflight loss counts, cancellation.3// Author: Simon-Pierre Boucher — contact@spboucher.ai4import ConvertCore5@testable import EngineSubtitle6import Foundation7import Testing89@Suite struct EncodingAndCleanupTests {10 /// Ordinary sentences in the language each code page is used for.11 static let samples: [(SubtitleEncoding, String)] = [12 (.windows1252, "Où est-ce que tu étais hier soir ? J'ai cherché partout, même à côté du café. Ça m'inquiète, tu sais. Voilà déjà trois heures passées."),13 (.windows1250, "Dobrý večer, přátelé. Proč jste včera nepřišli na večeři? Čekali jsme vás až do půlnoci. Żółć, gęś, łódź i źrebię."),14 (.windows1251, "Добрый вечер, друзья. Почему вы вчера не пришли на ужин? Мы ждали вас до полуночи. Я очень волновался, честное слово."),15 (.windows1253, "Καλησπέρα, φίλοι μου. Γιατί δεν ήρθατε χθες στο δείπνο; Σας περιμέναμε μέχρι τα μεσάνυχτα. Ανησύχησα πολύ."),16 (.windows1254, "İyi akşamlar arkadaşlar. Dün akşam yemeğe neden gelmediniz? Sizi gece yarısına kadar bekledik. Çok üzüldüm, değil mi şimdi?"),17 (.windows1255, "ערב טוב חברים. למה לא באתם אתמול לארוחת הערב? חיכינו לכם עד חצות. דאגתי מאוד, באמת."),18 (.windows1256, "مساء الخير يا أصدقائي. لماذا لم تأتوا أمس إلى العشاء؟ انتظرناكم حتى منتصف الليل. كنت قلقا جدا."),19 (.koi8r, "Добрый вечер, друзья. Почему вы вчера не пришли на ужин? Мы ждали вас до полуночи. Я очень волновался, честное слово."),20 (.shiftJIS, "こんばんは、みなさん。昨日の夕食にどうして来なかったのですか。私たちは真夜中まで待っていました。とても心配しました。"),21 (.gbk, "晚上好,朋友们。你们昨天为什么没有来吃晚饭?我们一直等到半夜。我真的非常担心,后来才知道发生了什么事情。"),22 (.big5, "晚安,朋友們。你們昨天為什麼沒有來吃晚餐?我們一直等到半夜。我真的非常擔心,後來才知道發生了什麼事情。"),23 (.eucKR, "안녕하세요 여러분. 어제 저녁 식사에 왜 오지 않았습니까? 우리는 자정까지 기다렸습니다. 정말 많이 걱정했습니다."),24 ]2526 @Test(arguments: samples.indices)27 func legacyCodePagesAreDetected(index: Int) throws {28 let (encoding, sentence) = Self.samples[index]29 let srt = "1\n00:00:01,000 --> 00:00:04,000\n\(sentence)\n\n2\n00:00:05,000 --> 00:00:08,000\n\(sentence)\n"30 let data = try #require(srt.data(using: encoding.foundation))31 let (text, detected) = EncodingDetector.decode(data)32 #expect(detected.encoding == encoding, "detected \(detected.encoding.rawValue), ranking: \(EncodingDetector.rank([UInt8](data)).prefix(4).map { "\($0.encoding.rawValue)=\(Int($0.score))" })")33 #expect(text == srt)34 #expect(detected.evidence == .heuristic && detected.confidence < 1 && detected.confidence > 0.2) // a guess is never reported as certain35 let doc = try SubtitleParser.parse(data: data, format: .srt)36 #expect(doc.cues[0].plainText == sentence)37 }3839 @Test func isoTwinsDecodeToTheSameTextWhenTheirBytesAgree() throws {40 // ISO-8859-1 text without bytes 0x80–0x9F is byte-identical in Windows-1252: reported as Windows-1252.41 let latin1 = try #require("Voilà une phrase très française, déjà vue à l'été.".data(using: .isoLatin1))42 let (text, detected) = EncodingDetector.decode(latin1)43 #expect(detected.encoding == .windows1252 && text.contains("très française"))44 // ISO-8859-2 differs from Windows-1250 for š, ž, ť…: the right one wins on Czech text.45 let czech = "Příliš žluťoučký kůň úpěl ďábelské ódy. Šest žížal šilhalo na žižkovské věže."46 let (t2, d2) = EncodingDetector.decode(try #require(czech.data(using: .isoLatin2)))47 #expect(d2.encoding == .iso8859_2 && t2 == czech)48 let (t3, d3) = EncodingDetector.decode(try #require(czech.data(using: .windowsCP1250)))49 #expect(d3.encoding == .windows1250 && t3 == czech)50 }5152 @Test func unicodeEncodingsAreRecognisedWithCertainty() throws {53 let s = "1\n00:00:01,000 --> 00:00:02,000\nÇa — 日本語 — Привет\n"54 var d = EncodingDetector.decode(Data(s.utf8)).detected55 #expect(d.encoding == .utf8 && d.evidence == .validUTF8 && d.confidence == 1 && !d.hadBOM)56 d = EncodingDetector.decode(Data([0xEF, 0xBB, 0xBF]) + Data(s.utf8)).detected57 #expect(d.evidence == .byteOrderMark && d.hadBOM)58 d = EncodingDetector.decode(Data("plain ascii".utf8)).detected59 #expect(d.evidence == .ascii && d.encoding == .utf8)60 for (bom, enc, e) in [([UInt8]([0xFF, 0xFE]), String.Encoding.utf16LittleEndian, SubtitleEncoding.utf16LE), ([0xFE, 0xFF], .utf16BigEndian, .utf16BE)] {61 let body = try #require(s.data(using: enc))62 let (text, with) = EncodingDetector.decode(Data(bom) + body)63 #expect(with.encoding == e && with.evidence == .byteOrderMark && text == s)64 let ascii = try #require("1\n00:00:01,000 --> 00:00:02,000\nNo BOM here, only Latin letters\n".data(using: enc))65 let without = EncodingDetector.decode(ascii).detected66 #expect(without.encoding == e && without.evidence == .utf16Pattern && without.confidence < 1)67 }68 }6970 @Test func forcedEncodingOverridesDetectionAndCountsWhatCannotBeDecoded() async throws {71 let scratch = try Scratch(); defer { scratch.cleanUp() }72 let text = "1\n00:00:01,000 --> 00:00:02,000\nŻółć\n"73 let source = try scratch.file("pl.srt", data: try #require(text.data(using: .windowsCP1250)))74 let forced = try await convert(source, .srt, .vtt, options: ["encoding.input": .string("windows-1250")], in: scratch)75 #expect(forced.text.contains("Żółć") && !forced.warnings.has("encoding-guessed"))76 let wrong = try await convert(source, .srt, .vtt, options: ["encoding.input": .string("windows-1251")], in: scratch)77 #expect(!wrong.text.contains("Żółć")) // the user's choice is obeyed, not second-guessed78 let auto = try await convert(source, .srt, .vtt, in: scratch)79 #expect(auto.warnings.has("encoding-guessed"))80 let bad = try scratch.file("bad.srt", data: Data("1\n00:00:01,000 --> 00:00:02,000\nab".utf8) + Data([0xFF, 0xFE, 0x41]) + Data("\n".utf8))81 let lossy = try await convert(bad, .srt, .vtt, options: ["encoding.input": .string("utf-8")], in: scratch)82 #expect(lossy.warnings.warnings.contains { $0.code == "encoding-undecodable" && $0.severity == .dataLoss && $0.message.hasPrefix("2 bytes") })83 }8485 @Test(arguments: [("utf-8", false), ("utf-8", true), ("utf-16le", true), ("utf-16be", true), ("windows-1252", false), ("windows-1251", false), ("shift_jis", false)])86 func outputEncodingsAreWrittenAndStillValidate(name: String, bom: Bool) async throws {87 let scratch = try Scratch(); defer { scratch.cleanUp() }88 let source = try scratch.file("a.srt", "1\n00:00:01,000 --> 00:00:02,000\nCafé déjà vu\n\n2\n00:00:03,000 --> 00:00:04,000\nПривет 日本\n")89 for destination in [FormatID.srt, .vtt, .ass, .sub] {90 let op: String? = destination == .srt ? SubtitleEngine.retimeOperation : nil91 let out = try await convert(source, .srt, destination, operation: op, options: ["encoding.output": .string(name), "encoding.bom": .bool(bom), "newline": .string("crlf")], in: scratch)92 for check in out.checks { #expect(check.passed, "\(name) \(destination): \(check.name) — \(check.detail)") }93 let data = try Data(contentsOf: out.url)94 let encoding = try #require(SubtitleEncoding(rawValue: name))95 if bom, name == "utf-8" { #expect(data.starts(with: [0xEF, 0xBB, 0xBF])) }96 if name == "utf-16le" { #expect(data.starts(with: [0xFF, 0xFE])) }97 let decoded = EncodingDetector.decode(data, forced: encoding).text98 #expect(decoded.contains("\r\n") && !decoded.replacingOccurrences(of: "\r\n", with: "").contains("\n"))99 let lossyExpected = name == "windows-1252" || name == "windows-1251" || name == "shift_jis"100 #expect(out.warnings.has("characters-unrepresentable") == lossyExpected)101 if name == "windows-1252" { #expect(decoded.contains("Café déjà vu") && decoded.contains("?????? ??")) }102 }103 }104105 @Test func hearingImpairedAnnotationsAreRemovedOnlyWhenAsked() async throws {106 let scratch = try Scratch(); defer { scratch.cleanUp() }107 let srt = """108 1109 00:00:01,000 --> 00:00:02,000110 [door slams]111112 2113 00:00:03,000 --> 00:00:04,000114 JOHN: Where are you? (sighs)115116 3117 00:00:05,000 --> 00:00:06,000118 - MRS. O'NEIL: <i>Here.</i>119 - [laughing] Not there.120121 4122 00:00:07,000 --> 00:00:08,000123 ♪ ♪124125 5126 00:00:09,000 --> 00:00:10,000127 Note: this stays. Time: 10:30 (really) matters?128129 6130 00:00:11,000 --> 00:00:12,000131 NARRATOR:132 <b>Once</b> upon a time133134 """135 let source = try scratch.file("hi.srt", srt)136 let kept = try await convert(source, .srt, .srt, operation: SubtitleEngine.cleanOperation, in: scratch)137 #expect(kept.text == srt + "\n")138 let cleaned = try await convert(source, .srt, .srt, operation: SubtitleEngine.cleanOperation, options: ["clean.hearing": .bool(true)], in: scratch)139 let doc = try cleaned.document(.srt)140 #expect(doc.cues.map(\.plainText) == ["Where are you?", "- Here.\n- Not there.", "Note: this stays. Time: 10:30 matters?", "Once upon a time"])141 #expect(doc.cues.map(\.startMs) == [3000, 5000, 9000, 11_000])142 #expect(cleaned.text.contains("- <i>Here.</i>") && cleaned.text.contains("<b>Once</b> upon a time")) // inline styles of what remains survive143 #expect(cleaned.warnings.warnings.contains { $0.code == "empty-removed" && $0.message.hasPrefix("2 cues") })144 for check in cleaned.checks { #expect(check.passed, "\(check.name) \(check.detail)") }145 let keptEmpty = try await SubtitlePipeline.run(url: source, from: .srt, to: .vtt, settings: { var s = SubtitleSettings(); s.stripHearingImpaired = true; s.removeEmpty = false; return s }())146 #expect(keptEmpty.cues.count == 6)147 }148149 @Test func formattingWhitespaceOcrAndWrapOptions() throws {150 let srt = "1\n00:00:01,000 --> 00:00:02,000\n{\\an8}<b>Bold</b> <font color=\"#00FF00\">green</font> \n\n2\n00:00:03,000 --> 00:00:04,000\nl'm sure l will. He|lo, ''quoted''. All well, lily.\n\n3\n00:00:05,000 --> 00:00:06,000\nThis is a fairly long subtitle line that should be wrapped\nonto several lines of bounded length\n\n4\n00:00:07,000 --> 00:00:08,000\n- First speaker talks here\n- Second speaker\n"151 let doc = try SubtitleParser.parse(data: Data(srt.utf8), format: .srt)152 var s = SubtitleSettings()153 s.stripFormatting = true; s.collapseWhitespace = true154 var r = SubtitlePipeline.run(document: doc, to: .srt, settings: s)155 #expect(String(decoding: r.data, as: UTF8.self).hasPrefix("1\n00:00:01,000 --> 00:00:02,000\nBold green\n\n"))156 #expect(r.transform.formattingStripped == 1 && r.cues[0].alignment == nil)157 s = SubtitleSettings(); s.fixOCR = true158 r = SubtitlePipeline.run(document: doc, to: .srt, settings: s)159 #expect(r.cues[1].plainText == "I'm sure I will. HeIlo, \"quoted\". All well, lily." && r.transform.ocrEdits == 1)160 #expect(SubtitlePipeline.run(document: doc, to: .srt, settings: .init()).cues[1].plainText.hasPrefix("l'm sure l will.")) // off by default161 s = SubtitleSettings(); s.wrapLength = 30162 r = SubtitlePipeline.run(document: doc, to: .srt, settings: s)163 let lines = r.cues[2].plainText.components(separatedBy: "\n")164 #expect(lines.count == 4 && lines.allSatisfy { $0.count <= 30 })165 #expect(lines.joined(separator: " ") == "This is a fairly long subtitle line that should be wrapped onto several lines of bounded length")166 #expect(r.cues[3].plainText == "- First speaker talks here\n- Second speaker")167 #expect(r.transform.rewrapped == 2)168 }169170 @Test func preflightCountsWhatTheFileWillLose() async throws {171 let scratch = try Scratch(); defer { scratch.cleanUp() }172 let source = try scratch.file("f.ass", Fixtures.ass(ssa: false))173 let engine = SubtitleEngine()174 func codes(_ to: FormatID, _ options: [String: OptionValue] = [:]) -> [String: String] {175 let w = engine.preflight(.init(source: source, sourceFormat: .ass, destinationFormat: to, options: ConversionOptions(options)), inspection: nil)176 return Dictionary(w.map { ($0.code, $0.message) }, uniquingKeysWith: { a, _ in a })177 }178 let toSRT = codes(.srt)179 #expect(toSRT["style-definitions-dropped"]?.hasPrefix("2 style definitions") == true)180 #expect(toSRT["position-dropped"]?.hasPrefix("1 cue ") == true)181 #expect(toSRT["named-styles-dropped"]?.hasPrefix("1 cue ") == true)182 #expect(toSRT["events-dropped"]?.hasPrefix("1 comment line") == true)183 #expect(codes(.srt, ["srt.alignment": .bool(true)])["position-dropped"] == nil)184 let toVTT = codes(.vtt)185 #expect(toVTT["position-dropped"] == nil && toVTT["style-definitions-dropped"] != nil)186 #expect(codes(.sub)["framerate-assumed-output"] != nil)187 #expect(codes(.ssa)["style-definitions-dropped"] == nil)188 // What preflight announces is what convert then reports.189 let converted = try await convert(source, .ass, .srt, in: scratch)190 #expect(Set(converted.warnings.warnings.map(\.code)) == Set(toSRT.keys))191 // A plain SRT loses nothing going to VTT: no warning at all.192 let plain = try scratch.file("p.srt", "1\n00:00:01,000 --> 00:00:02,000\nplain\n")193 #expect(engine.preflight(.init(source: plain, sourceFormat: .srt, destinationFormat: .vtt), inspection: nil).isEmpty)194 }195196 @Test func cancellationAndUnsupportedRequestsThrow() async throws {197 let scratch = try Scratch(); defer { scratch.cleanUp() }198 let source = try scratch.file("a.srt", Fixtures.srt)199 let work = try scratch.work()200 let task = Task {201 try await Task.sleep(for: .seconds(30))202 return try await SubtitleEngine().convert(.init(source: source, sourceFormat: .srt, destinationFormat: .vtt), context: .init(workDirectory: work))203 }204 task.cancel()205 await #expect(throws: CancellationError.self) { _ = try await task.value }206 let cancelled = Task { () -> ConversionOutput in207 withUnsafeCurrentTask { $0?.cancel() }208 return try await SubtitleEngine().convert(.init(source: source, sourceFormat: .srt, destinationFormat: .vtt), context: .init(workDirectory: work))209 }210 await #expect(throws: CancellationError.self) { _ = try await cancelled.value }211 #expect((try? FileManager.default.contentsOfDirectory(atPath: work.path))?.isEmpty == true)212 await #expect(throws: ConversionError.unsupportedPath(source: .srt, destination: .pdf)) {213 _ = try await SubtitleEngine().convert(.init(source: source, sourceFormat: .srt, destinationFormat: .pdf), context: .init(workDirectory: work))214 }215 await #expect(throws: ConversionError.unsupportedPath(source: .srt, destination: .vtt)) {216 _ = try await SubtitleEngine().convert(.init(source: source, sourceFormat: .srt, destinationFormat: .vtt, operation: SubtitleEngine.retimeOperation), context: .init(workDirectory: work))217 }218 // Clean-up that removes everything is an error, not an empty file.219 let onlyNoise = try scratch.file("n.srt", "1\n00:00:01,000 --> 00:00:02,000\n[music]\n")220 await #expect(throws: ConversionError.self) {221 _ = try await SubtitleEngine().convert(.init(source: onlyNoise, sourceFormat: .srt, destinationFormat: .srt, operation: SubtitleEngine.cleanOperation,222 options: ConversionOptions(["clean.hearing": .bool(true)])), context: .init(workDirectory: work))223 }224 }225}226