SPB Git

Repository Public

PlainTextPDFTests.swift 13.0 KB 189 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocFormatsTests/PlainTextPDFTests.swift3// Description: Plain text (encoding detection, line endings, export of lists / tables / notes) and PDF text import4//              (paragraph rebuilding, page breaks, locked and text-less PDFs).5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Testing9import Foundation10import DocModel11import DocContainer12@testable import DocFormats1314@Suite struct PlainTextTests {15    private func utf16(_ s: String, little: Bool, bom: Bool) -> Data {16        var d = Data(bom ? (little ? [0xFF, 0xFE] : [0xFE, 0xFF]) : [])17        for u in s.utf16 { d.append(contentsOf: little ? [UInt8(u & 0xFF), UInt8(u >> 8)] : [UInt8(u >> 8), UInt8(u & 0xFF)]) }18        return d19    }2021    @Test func encodingDetection() {22        let text = "Élan café — 中文 😀"23        func decoded(_ d: Data) -> (String, String, Bool) { let r = TextDecoder.decode(d); return (r.text, r.encoding, r.guessed) }24        #expect(decoded(Data(text.utf8)) == (text, "UTF-8", false))25        #expect(decoded(Data([0xEF, 0xBB, 0xBF]) + Data(text.utf8)) == (text, "UTF-8", false))26        #expect(decoded(utf16(text, little: true, bom: true)) == (text, "UTF-16LE", false))27        #expect(decoded(utf16(text, little: false, bom: true)) == (text, "UTF-16BE", false))28        // BOM-less UTF-16 is recognised from the position of its zero bytes.29        let ascii = "Plain ASCII text stored as UTF-16 without any byte-order mark.\n"30        #expect(decoded(utf16(ascii, little: true, bom: false)) == (ascii, "UTF-16LE", true))31        #expect(decoded(utf16(ascii, little: false, bom: false)) == (ascii, "UTF-16BE", true))32        // UTF-32 with BOM.33        var utf32 = Data([0xFF, 0xFE, 0x00, 0x00])34        for u in "A😀".unicodeScalars { withUnsafeBytes(of: u.value.littleEndian) { utf32.append(contentsOf: $0) } }35        #expect(decoded(utf32) == ("A😀", "UTF-32LE", false))36        // Legacy 8-bit text: Windows-1252, including its typographic characters in 0x80...0x9F.37        let cp1252 = Data([0x93, 0x51, 0x75, 0x6F, 0x74, 0x65, 0x94, 0x20, 0x63, 0x61, 0x66, 0xE9, 0x20, 0x80, 0x35])38        #expect(decoded(cp1252) == ("\u{201C}Quote\u{201D} café €5", "Windows-1252", true))39        // Bytes undefined in Windows-1252 fall back to ISO-8859-1 rather than failing.40        let latin1 = TextDecoder.decode(Data([0x41, 0x81, 0xE9, 0x8D]))41        #expect(latin1.guessed && latin1.text.hasPrefix("A") && latin1.text.contains("é") && latin1.text.unicodeScalars.count == 4)42        // Mostly valid UTF-8 with one damaged byte is repaired, not misread as an 8-bit encoding.43        var damaged = Data("Déjà vu, naïve façade, crème brûlée".utf8)44        damaged.append(0xFF)45        let repaired = TextDecoder.decode(damaged)46        #expect(repaired.encoding == "UTF-8" && repaired.replacedInvalidBytes && repaired.text == "Déjà vu, naïve façade, crème brûlée\u{FFFD}")47        // Odd-length and lone-surrogate UTF-16.48        let lone = TextDecoder.decode(Data([0xFF, 0xFE, 0x41, 0x00, 0x00, 0xD8, 0x42, 0x00, 0x43]))49        #expect(lone.text == "A\u{FFFD}B" && lone.replacedInvalidBytes)50        #expect(TextDecoder.decode(Data()).text == "")51    }5253    @Test func importSplitsLinesAndNormalisesEndings() throws {54        let result = try PlainText.read(Data("first\r\nsecond\rthird\n\nafter blank\twith tab\u{0C}new page\nnul\u{0}gone\u{FFFC}\n".utf8))55        #expect(Describe.blocks(result.document) == ["Normal: first", "Normal: second", "Normal: third", "Normal: ", "Normal: after blank\twith tab", "PAGEBREAK", "Normal: new page", "Normal: nulgone\u{FFFD}"])56        #expect(result.warnings.map(\.feature) == ["Control characters"])57        #expect(Describe.blocks(try PlainText.read(Data()).document) == ["Normal: "])58        #expect(Describe.blocks(try PlainText.read(Data("no newline".utf8)).document) == ["Normal: no newline"])59        #expect(Describe.blocks(try PlainText.read(Data("\n".utf8)).document) == ["Normal: "])60        #expect(Describe.blocks(try PlainText.read(Data("\n\n".utf8)).document) == ["Normal: ", "Normal: "])61        // A legacy encoding is reported to the user.62        let legacy = try PlainText.read(Data([0x63, 0x61, 0x66, 0xE9]))63        #expect(legacy.document.body[0].paragraph?.visibleText == "café")64        #expect(legacy.warnings.first?.severity == .approximated)65        #expect(legacy.warnings.first?.detail.contains("Windows-1252") == true)66        // No paragraph ever holds an inline object or formatting.67        #expect(result.document.body.compactMap(\.paragraph).allSatisfy { $0.runs.allSatisfy { $0.inline == nil && $0.props == .none } })68    }6970    @Test func roundTripIsExactForPlainDocuments() throws {71        let text = "Title\n\nFirst paragraph, with ünïcödé 中文 العربية 👩‍👩‍👧‍👦.\n\tIndented with a tab\n\n\nTwo blank lines above\u{0C}After a page break"72        let doc = try PlainText.read(Data(text.utf8)).document73        let exported = PlainText.export(doc)74        #expect(String(decoding: exported.data, as: UTF8.self) == text)75        #expect(exported.warnings.map(\.feature) == ["Encoding"])76        #expect(try PlainText.read(exported.data).document.body.count == doc.body.count)77    }7879    @Test func exportOfARichDocument() {80        let result = PlainText.export(Fixture.rich())81        let text = String(decoding: result.data, as: UTF8.self)82        let lines = text.components(separatedBy: "\n")83        #expect(lines[0] == "Rich fixture")84        #expect(lines[1...3] == ["Sommaire", "Introduction", "    Second section"])85        #expect(lines.contains("Plain bold italic link styled inserted[A blue box][Photo]Notee^{i\\pi}+1=0[1][i][see @knuth1984, p. 4; -@doe:2020, passim]Figure 1?2026Rich « fixture »Ada Lovelace end"))86        #expect(lines.contains("second line\ttabbed"))87        #expect(lines.contains("Figure 1: A caption"))88        // Lists carry their computed labels, nested levels are indented.89        #expect(lines.contains("• First bullet") && lines.contains("    ◦ Nested bullet") && lines.contains("1. One") && lines.contains("    1.1. One point one"))90        // Tables: one line per row, cells separated by tabs, merged-away cells empty.91        #expect(lines.contains("Header A+B\tTall") && lines.contains("a1\tb1\t"))92        // The page break and the odd-page section break are form feeds (the column break is not); notes are appended with their marks.93        #expect(text.contains("\na2\tline one line two\t42\u{0C}\u{0C}Second section\n"))94        #expect(text.hasSuffix("Références\n\n[1] A footnote with emphasis.\n[i] An endnote.\nSecond paragraph."))95        let features = Set(result.warnings.map(\.feature))96        for expected in ["Formatting", "Images", "Links", "Tables", "Equations", "Footnotes and endnotes", "Comments", "Tracked changes", "Generated lists", "Encoding"] {97            #expect(features.contains(expected), "missing warning: \(expected)")98        }99        #expect(result.warnings.first { $0.feature == "Formatting" }?.severity == .dropped)100    }101102    @Test func oversizedInputIsRefused() {103        let big = Data(count: FormatLimits.maxTextBytes + 1)104        #expect(throws: FormatError.self) { _ = try PlainText.read(big) }105        #expect(PlainText.importDocument(big).warnings.first?.feature == "Unreadable Plain Text")106    }107}108109@Suite struct PDFTextImportTests {110    @Test func textIsRebuiltIntoParagraphsWithPageBreaks() throws {111        let pdf = Fixture.pdf(pages: [112            ["Annual Report", "",113             "This is the first paragraph of the document, set on lines of about the same",114             "length, as a justified paragraph would be. A word is hyphen-",115             "ated across two lines here.",116             "The second paragraph follows without any blank line, but the previous line",117             "ended a sentence and was short.", "",118             "A third paragraph, separated by a blank line."],119            ["Second page heading", "", "Text on the second page."],120        ], title: "PDF Title")121        let result = try PDFTextImporter.read(pdf)122        let lines = Describe.blocks(result.document)123        #expect(lines.first == "Normal: Annual Report")124        #expect(lines.contains("PAGEBREAK"))125        #expect(lines.last == "Normal: Text on the second page.")126        let breakIndex = try #require(lines.firstIndex(of: "PAGEBREAK"))127        #expect(lines[breakIndex + 1] == "Normal: Second page heading")128        // Lines of one paragraph are joined; the hyphenated word is whole again.129        let all = Describe.texts(result.document)130        #expect(all.contains { $0.hasPrefix("This is the first paragraph") && $0.hasSuffix("A word is hyphenated across two lines here.") })131        #expect(all.contains { $0.hasPrefix("The second paragraph follows") && $0.hasSuffix("was short.") })132        #expect(all.contains("A third paragraph, separated by a blank line."))133        #expect(result.document.meta.title == "PDF Title")134        #expect(result.document.pageSetup.paper == .letter)135        // The loss of layout is always stated.136        let layout = try #require(result.warnings.first { $0.feature == "PDF layout" })137        #expect(layout.severity == .approximated)138        #expect(layout.detail.contains("layout") && layout.detail.contains("fonts") && layout.detail.contains("images") && layout.detail.contains("tables"))139        #expect(result.warnings.contains { $0.feature == "Hyphenation" && $0.detail.hasPrefix("1 word") })140        #expect(result.document.body.compactMap(\.paragraph).allSatisfy { $0.style == StyleID.normal && $0.runs.allSatisfy { $0.props == .none } })141    }142143    @Test func paragraphHeuristics() {144        var joined = 0145        #expect(PDFTextImporter.rebuildParagraphs("", joinedHyphens: &joined).isEmpty)146        #expect(PDFTextImporter.rebuildParagraphs("  \n\n \n", joinedHyphens: &joined).isEmpty)147        #expect(PDFTextImporter.rebuildParagraphs("one line", joinedHyphens: &joined) == ["one line"])148        #expect(PDFTextImporter.rebuildParagraphs("a first line that is quite long indeed and goes on\nand a second one of the same length, more or less\n\nnext", joinedHyphens: &joined)149                == ["a first line that is quite long indeed and goes on and a second one of the same length, more or less", "next"])150        // A compound word broken after its hyphen keeps the hyphen when the next line starts with a capital.151        #expect(PDFTextImporter.rebuildParagraphs("a long first line that happens to end with the compound Franco-\nGerman border, and then the long line that follows it here", joinedHyphens: &joined)152                == ["a long first line that happens to end with the compound Franco- German border, and then the long line that follows it here"])153        #expect(joined == 0)154        #expect(PDFTextImporter.rebuildParagraphs("inter-\nnational", joinedHyphens: &joined) == ["international"])155        #expect(joined == 1)156    }157158    @Test func passwordProtectedPDFIsRefusedClearly() {159        let locked = Fixture.pdf(pages: [["Secret text"]], password: "hunter2")160        do { _ = try PDFTextImporter.read(locked); Issue.record("a locked PDF must not be read") } catch let e as FormatError {161            guard case .unreadable(let why) = e else { Issue.record("\(e)"); return }162            #expect(why.contains("password"))163        } catch { Issue.record("\(error)") }164        let result = PDFTextImporter.importDocument(locked)165        #expect(result.warnings.count == 1 && result.warnings[0].severity == .dropped && result.warnings[0].detail.contains("password-protected"))166        #expect(Describe.texts(result.document) == [""])           // nothing leaked from the locked file167    }168169    @Test func scannedPDFWithoutTextLayer() throws {170        let scan = Fixture.pdf(pages: [[], []])171        do { _ = try PDFTextImporter.read(scan); Issue.record("a text-less PDF should be refused") } catch let e as FormatError {172            guard case .unreadable(let why) = e else { Issue.record("\(e)"); return }173            #expect(why.contains("no text layer") && why.contains("OCR"))174        } catch { Issue.record("\(error)") }175        // Some pages with text, some without: imported, with the number of empty pages reported.176        let mixed = try PDFTextImporter.read(Fixture.pdf(pages: [["Readable page."], [], ["Another readable page."]]))177        #expect(Describe.blocks(mixed.document) == ["Normal: Readable page.", "PAGEBREAK", "PAGEBREAK", "Normal: Another readable page."])178        #expect(mixed.warnings.contains { $0.feature == "Pages without text" && $0.severity == .dropped && $0.detail.hasPrefix("1 page has") })179    }180181    @Test func notAPDF() {182        #expect(throws: FormatError.unreadable("the file is not a PDF document")) { _ = try PDFTextImporter.read(Data("Hello".utf8)) }183        #expect(throws: FormatError.unreadable("the file is not a PDF document")) { _ = try PDFTextImporter.read(Data()) }184        #expect(throws: FormatError.self) { _ = try PDFTextImporter.read(Data("%PDF-1.7\ngarbage that is not a PDF body".utf8)) }185        let truncated = Fixture.pdf(pages: [["Some text"]]).prefix(200)186        #expect(throws: FormatError.self) { _ = try PDFTextImporter.read(Data(truncated)) }187    }188}189
Files main
Menu

Theme