SPB Git

Repository Public License

EncodingAndCleanupTests.swift 16.6 KB 226 lines swift

1// ORIX Convert — Packages/EngineSubtitle/Tests/EngineSubtitleTests/EncodingAndCleanupTests.swift2// Description: Encoding detection and output, text clean-up options, preflight loss counts, cancellation.3// Author: Simon-Pierre Boucher — contact@spboucher.ai4import ConvertCore5@testable import EngineSubtitle6import Foundation7import Testing89@Suite struct EncodingAndCleanupTests {10    /// Ordinary sentences in the language each code page is used for.11    static let samples: [(SubtitleEncoding, String)] = [12        (.windows1252, "Où est-ce que tu étais hier soir ? J'ai cherché partout, même à côté du café. Ça m'inquiète, tu sais. Voilà déjà trois heures passées."),13        (.windows1250, "Dobrý večer, přátelé. Proč jste včera nepřišli na večeři? Čekali jsme vás až do půlnoci. Żółć, gęś, łódź i źrebię."),14        (.windows1251, "Добрый вечер, друзья. Почему вы вчера не пришли на ужин? Мы ждали вас до полуночи. Я очень волновался, честное слово."),15        (.windows1253, "Καλησπέρα, φίλοι μου. Γιατί δεν ήρθατε χθες στο δείπνο; Σας περιμέναμε μέχρι τα μεσάνυχτα. Ανησύχησα πολύ."),16        (.windows1254, "İyi akşamlar arkadaşlar. Dün akşam yemeğe neden gelmediniz? Sizi gece yarısına kadar bekledik. Çok üzüldüm, değil mi şimdi?"),17        (.windows1255, "ערב טוב חברים. למה לא באתם אתמול לארוחת הערב? חיכינו לכם עד חצות. דאגתי מאוד, באמת."),18        (.windows1256, "مساء الخير يا أصدقائي. لماذا لم تأتوا أمس إلى العشاء؟ انتظرناكم حتى منتصف الليل. كنت قلقا جدا."),19        (.koi8r, "Добрый вечер, друзья. Почему вы вчера не пришли на ужин? Мы ждали вас до полуночи. Я очень волновался, честное слово."),20        (.shiftJIS, "こんばんは、みなさん。昨日の夕食にどうして来なかったのですか。私たちは真夜中まで待っていました。とても心配しました。"),21        (.gbk, "晚上好,朋友们。你们昨天为什么没有来吃晚饭?我们一直等到半夜。我真的非常担心,后来才知道发生了什么事情。"),22        (.big5, "晚安,朋友們。你們昨天為什麼沒有來吃晚餐?我們一直等到半夜。我真的非常擔心,後來才知道發生了什麼事情。"),23        (.eucKR, "안녕하세요 여러분. 어제 저녁 식사에 왜 오지 않았습니까? 우리는 자정까지 기다렸습니다. 정말 많이 걱정했습니다."),24    ]2526    @Test(arguments: samples.indices)27    func legacyCodePagesAreDetected(index: Int) throws {28        let (encoding, sentence) = Self.samples[index]29        let srt = "1\n00:00:01,000 --> 00:00:04,000\n\(sentence)\n\n2\n00:00:05,000 --> 00:00:08,000\n\(sentence)\n"30        let data = try #require(srt.data(using: encoding.foundation))31        let (text, detected) = EncodingDetector.decode(data)32        #expect(detected.encoding == encoding, "detected \(detected.encoding.rawValue), ranking: \(EncodingDetector.rank([UInt8](data)).prefix(4).map { "\($0.encoding.rawValue)=\(Int($0.score))" })")33        #expect(text == srt)34        #expect(detected.evidence == .heuristic && detected.confidence < 1 && detected.confidence > 0.2)      // a guess is never reported as certain35        let doc = try SubtitleParser.parse(data: data, format: .srt)36        #expect(doc.cues[0].plainText == sentence)37    }3839    @Test func isoTwinsDecodeToTheSameTextWhenTheirBytesAgree() throws {40        // ISO-8859-1 text without bytes 0x80–0x9F is byte-identical in Windows-1252: reported as Windows-1252.41        let latin1 = try #require("Voilà une phrase très française, déjà vue à l'été.".data(using: .isoLatin1))42        let (text, detected) = EncodingDetector.decode(latin1)43        #expect(detected.encoding == .windows1252 && text.contains("très française"))44        // ISO-8859-2 differs from Windows-1250 for š, ž, ť…: the right one wins on Czech text.45        let czech = "Příliš žluťoučký kůň úpěl ďábelské ódy. Šest žížal šilhalo na žižkovské věže."46        let (t2, d2) = EncodingDetector.decode(try #require(czech.data(using: .isoLatin2)))47        #expect(d2.encoding == .iso8859_2 && t2 == czech)48        let (t3, d3) = EncodingDetector.decode(try #require(czech.data(using: .windowsCP1250)))49        #expect(d3.encoding == .windows1250 && t3 == czech)50    }5152    @Test func unicodeEncodingsAreRecognisedWithCertainty() throws {53        let s = "1\n00:00:01,000 --> 00:00:02,000\nÇa — 日本語 — Привет\n"54        var d = EncodingDetector.decode(Data(s.utf8)).detected55        #expect(d.encoding == .utf8 && d.evidence == .validUTF8 && d.confidence == 1 && !d.hadBOM)56        d = EncodingDetector.decode(Data([0xEF, 0xBB, 0xBF]) + Data(s.utf8)).detected57        #expect(d.evidence == .byteOrderMark && d.hadBOM)58        d = EncodingDetector.decode(Data("plain ascii".utf8)).detected59        #expect(d.evidence == .ascii && d.encoding == .utf8)60        for (bom, enc, e) in [([UInt8]([0xFF, 0xFE]), String.Encoding.utf16LittleEndian, SubtitleEncoding.utf16LE), ([0xFE, 0xFF], .utf16BigEndian, .utf16BE)] {61            let body = try #require(s.data(using: enc))62            let (text, with) = EncodingDetector.decode(Data(bom) + body)63            #expect(with.encoding == e && with.evidence == .byteOrderMark && text == s)64            let ascii = try #require("1\n00:00:01,000 --> 00:00:02,000\nNo BOM here, only Latin letters\n".data(using: enc))65            let without = EncodingDetector.decode(ascii).detected66            #expect(without.encoding == e && without.evidence == .utf16Pattern && without.confidence < 1)67        }68    }6970    @Test func forcedEncodingOverridesDetectionAndCountsWhatCannotBeDecoded() async throws {71        let scratch = try Scratch(); defer { scratch.cleanUp() }72        let text = "1\n00:00:01,000 --> 00:00:02,000\nŻółć\n"73        let source = try scratch.file("pl.srt", data: try #require(text.data(using: .windowsCP1250)))74        let forced = try await convert(source, .srt, .vtt, options: ["encoding.input": .string("windows-1250")], in: scratch)75        #expect(forced.text.contains("Żółć") && !forced.warnings.has("encoding-guessed"))76        let wrong = try await convert(source, .srt, .vtt, options: ["encoding.input": .string("windows-1251")], in: scratch)77        #expect(!wrong.text.contains("Żółć"))                                   // the user's choice is obeyed, not second-guessed78        let auto = try await convert(source, .srt, .vtt, in: scratch)79        #expect(auto.warnings.has("encoding-guessed"))80        let bad = try scratch.file("bad.srt", data: Data("1\n00:00:01,000 --> 00:00:02,000\nab".utf8) + Data([0xFF, 0xFE, 0x41]) + Data("\n".utf8))81        let lossy = try await convert(bad, .srt, .vtt, options: ["encoding.input": .string("utf-8")], in: scratch)82        #expect(lossy.warnings.warnings.contains { $0.code == "encoding-undecodable" && $0.severity == .dataLoss && $0.message.hasPrefix("2 bytes") })83    }8485    @Test(arguments: [("utf-8", false), ("utf-8", true), ("utf-16le", true), ("utf-16be", true), ("windows-1252", false), ("windows-1251", false), ("shift_jis", false)])86    func outputEncodingsAreWrittenAndStillValidate(name: String, bom: Bool) async throws {87        let scratch = try Scratch(); defer { scratch.cleanUp() }88        let source = try scratch.file("a.srt", "1\n00:00:01,000 --> 00:00:02,000\nCafé déjà vu\n\n2\n00:00:03,000 --> 00:00:04,000\nПривет 日本\n")89        for destination in [FormatID.srt, .vtt, .ass, .sub] {90            let op: String? = destination == .srt ? SubtitleEngine.retimeOperation : nil91            let out = try await convert(source, .srt, destination, operation: op, options: ["encoding.output": .string(name), "encoding.bom": .bool(bom), "newline": .string("crlf")], in: scratch)92            for check in out.checks { #expect(check.passed, "\(name) \(destination): \(check.name) — \(check.detail)") }93            let data = try Data(contentsOf: out.url)94            let encoding = try #require(SubtitleEncoding(rawValue: name))95            if bom, name == "utf-8" { #expect(data.starts(with: [0xEF, 0xBB, 0xBF])) }96            if name == "utf-16le" { #expect(data.starts(with: [0xFF, 0xFE])) }97            let decoded = EncodingDetector.decode(data, forced: encoding).text98            #expect(decoded.contains("\r\n") && !decoded.replacingOccurrences(of: "\r\n", with: "").contains("\n"))99            let lossyExpected = name == "windows-1252" || name == "windows-1251" || name == "shift_jis"100            #expect(out.warnings.has("characters-unrepresentable") == lossyExpected)101            if name == "windows-1252" { #expect(decoded.contains("Café déjà vu") && decoded.contains("?????? ??")) }102        }103    }104105    @Test func hearingImpairedAnnotationsAreRemovedOnlyWhenAsked() async throws {106        let scratch = try Scratch(); defer { scratch.cleanUp() }107        let srt = """108        1109        00:00:01,000 --> 00:00:02,000110        [door slams]111112        2113        00:00:03,000 --> 00:00:04,000114        JOHN: Where are you? (sighs)115116        3117        00:00:05,000 --> 00:00:06,000118        - MRS. O'NEIL: <i>Here.</i>119        - [laughing] Not there.120121        4122        00:00:07,000 --> 00:00:08,000123        ♪ ♪124125        5126        00:00:09,000 --> 00:00:10,000127        Note: this stays. Time: 10:30 (really) matters?128129        6130        00:00:11,000 --> 00:00:12,000131        NARRATOR:132        <b>Once</b> upon a time133134        """135        let source = try scratch.file("hi.srt", srt)136        let kept = try await convert(source, .srt, .srt, operation: SubtitleEngine.cleanOperation, in: scratch)137        #expect(kept.text == srt + "\n")138        let cleaned = try await convert(source, .srt, .srt, operation: SubtitleEngine.cleanOperation, options: ["clean.hearing": .bool(true)], in: scratch)139        let doc = try cleaned.document(.srt)140        #expect(doc.cues.map(\.plainText) == ["Where are you?", "- Here.\n- Not there.", "Note: this stays. Time: 10:30 matters?", "Once upon a time"])141        #expect(doc.cues.map(\.startMs) == [3000, 5000, 9000, 11_000])142        #expect(cleaned.text.contains("- <i>Here.</i>") && cleaned.text.contains("<b>Once</b> upon a time"))        // inline styles of what remains survive143        #expect(cleaned.warnings.warnings.contains { $0.code == "empty-removed" && $0.message.hasPrefix("2 cues") })144        for check in cleaned.checks { #expect(check.passed, "\(check.name) \(check.detail)") }145        let keptEmpty = try await SubtitlePipeline.run(url: source, from: .srt, to: .vtt, settings: { var s = SubtitleSettings(); s.stripHearingImpaired = true; s.removeEmpty = false; return s }())146        #expect(keptEmpty.cues.count == 6)147    }148149    @Test func formattingWhitespaceOcrAndWrapOptions() throws {150        let srt = "1\n00:00:01,000 --> 00:00:02,000\n{\\an8}<b>Bold</b>   <font color=\"#00FF00\">green</font>  \n\n2\n00:00:03,000 --> 00:00:04,000\nl'm sure l will. He|lo, ''quoted''. All well, lily.\n\n3\n00:00:05,000 --> 00:00:06,000\nThis is a fairly long subtitle line that should be wrapped\nonto several lines of bounded length\n\n4\n00:00:07,000 --> 00:00:08,000\n- First speaker talks here\n- Second speaker\n"151        let doc = try SubtitleParser.parse(data: Data(srt.utf8), format: .srt)152        var s = SubtitleSettings()153        s.stripFormatting = true; s.collapseWhitespace = true154        var r = SubtitlePipeline.run(document: doc, to: .srt, settings: s)155        #expect(String(decoding: r.data, as: UTF8.self).hasPrefix("1\n00:00:01,000 --> 00:00:02,000\nBold green\n\n"))156        #expect(r.transform.formattingStripped == 1 && r.cues[0].alignment == nil)157        s = SubtitleSettings(); s.fixOCR = true158        r = SubtitlePipeline.run(document: doc, to: .srt, settings: s)159        #expect(r.cues[1].plainText == "I'm sure I will. HeIlo, \"quoted\". All well, lily." && r.transform.ocrEdits == 1)160        #expect(SubtitlePipeline.run(document: doc, to: .srt, settings: .init()).cues[1].plainText.hasPrefix("l'm sure l will."))   // off by default161        s = SubtitleSettings(); s.wrapLength = 30162        r = SubtitlePipeline.run(document: doc, to: .srt, settings: s)163        let lines = r.cues[2].plainText.components(separatedBy: "\n")164        #expect(lines.count == 4 && lines.allSatisfy { $0.count <= 30 })165        #expect(lines.joined(separator: " ") == "This is a fairly long subtitle line that should be wrapped onto several lines of bounded length")166        #expect(r.cues[3].plainText == "- First speaker talks here\n- Second speaker")167        #expect(r.transform.rewrapped == 2)168    }169170    @Test func preflightCountsWhatTheFileWillLose() async throws {171        let scratch = try Scratch(); defer { scratch.cleanUp() }172        let source = try scratch.file("f.ass", Fixtures.ass(ssa: false))173        let engine = SubtitleEngine()174        func codes(_ to: FormatID, _ options: [String: OptionValue] = [:]) -> [String: String] {175            let w = engine.preflight(.init(source: source, sourceFormat: .ass, destinationFormat: to, options: ConversionOptions(options)), inspection: nil)176            return Dictionary(w.map { ($0.code, $0.message) }, uniquingKeysWith: { a, _ in a })177        }178        let toSRT = codes(.srt)179        #expect(toSRT["style-definitions-dropped"]?.hasPrefix("2 style definitions") == true)180        #expect(toSRT["position-dropped"]?.hasPrefix("1 cue ") == true)181        #expect(toSRT["named-styles-dropped"]?.hasPrefix("1 cue ") == true)182        #expect(toSRT["events-dropped"]?.hasPrefix("1 comment line") == true)183        #expect(codes(.srt, ["srt.alignment": .bool(true)])["position-dropped"] == nil)184        let toVTT = codes(.vtt)185        #expect(toVTT["position-dropped"] == nil && toVTT["style-definitions-dropped"] != nil)186        #expect(codes(.sub)["framerate-assumed-output"] != nil)187        #expect(codes(.ssa)["style-definitions-dropped"] == nil)188        // What preflight announces is what convert then reports.189        let converted = try await convert(source, .ass, .srt, in: scratch)190        #expect(Set(converted.warnings.warnings.map(\.code)) == Set(toSRT.keys))191        // A plain SRT loses nothing going to VTT: no warning at all.192        let plain = try scratch.file("p.srt", "1\n00:00:01,000 --> 00:00:02,000\nplain\n")193        #expect(engine.preflight(.init(source: plain, sourceFormat: .srt, destinationFormat: .vtt), inspection: nil).isEmpty)194    }195196    @Test func cancellationAndUnsupportedRequestsThrow() async throws {197        let scratch = try Scratch(); defer { scratch.cleanUp() }198        let source = try scratch.file("a.srt", Fixtures.srt)199        let work = try scratch.work()200        let task = Task {201            try await Task.sleep(for: .seconds(30))202            return try await SubtitleEngine().convert(.init(source: source, sourceFormat: .srt, destinationFormat: .vtt), context: .init(workDirectory: work))203        }204        task.cancel()205        await #expect(throws: CancellationError.self) { _ = try await task.value }206        let cancelled = Task { () -> ConversionOutput in207            withUnsafeCurrentTask { $0?.cancel() }208            return try await SubtitleEngine().convert(.init(source: source, sourceFormat: .srt, destinationFormat: .vtt), context: .init(workDirectory: work))209        }210        await #expect(throws: CancellationError.self) { _ = try await cancelled.value }211        #expect((try? FileManager.default.contentsOfDirectory(atPath: work.path))?.isEmpty == true)212        await #expect(throws: ConversionError.unsupportedPath(source: .srt, destination: .pdf)) {213            _ = try await SubtitleEngine().convert(.init(source: source, sourceFormat: .srt, destinationFormat: .pdf), context: .init(workDirectory: work))214        }215        await #expect(throws: ConversionError.unsupportedPath(source: .srt, destination: .vtt)) {216            _ = try await SubtitleEngine().convert(.init(source: source, sourceFormat: .srt, destinationFormat: .vtt, operation: SubtitleEngine.retimeOperation), context: .init(workDirectory: work))217        }218        // Clean-up that removes everything is an error, not an empty file.219        let onlyNoise = try scratch.file("n.srt", "1\n00:00:01,000 --> 00:00:02,000\n[music]\n")220        await #expect(throws: ConversionError.self) {221            _ = try await SubtitleEngine().convert(.init(source: onlyNoise, sourceFormat: .srt, destinationFormat: .srt, operation: SubtitleEngine.cleanOperation,222                                                         options: ConversionOptions(["clean.hearing": .bool(true)])), context: .init(workDirectory: work))223        }224    }225}226
Files main
Menu

Theme