Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocFormatsTests/PlainTextPDFTests.swift3// Description: Plain text (encoding detection, line endings, export of lists / tables / notes) and PDF text import4// (paragraph rebuilding, page breaks, locked and text-less PDFs).5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Testing9import Foundation10import DocModel11import DocContainer12@testable import DocFormats1314@Suite struct PlainTextTests {15 private func utf16(_ s: String, little: Bool, bom: Bool) -> Data {16 var d = Data(bom ? (little ? [0xFF, 0xFE] : [0xFE, 0xFF]) : [])17 for u in s.utf16 { d.append(contentsOf: little ? [UInt8(u & 0xFF), UInt8(u >> 8)] : [UInt8(u >> 8), UInt8(u & 0xFF)]) }18 return d19 }2021 @Test func encodingDetection() {22 let text = "Élan café — 中文 😀"23 func decoded(_ d: Data) -> (String, String, Bool) { let r = TextDecoder.decode(d); return (r.text, r.encoding, r.guessed) }24 #expect(decoded(Data(text.utf8)) == (text, "UTF-8", false))25 #expect(decoded(Data([0xEF, 0xBB, 0xBF]) + Data(text.utf8)) == (text, "UTF-8", false))26 #expect(decoded(utf16(text, little: true, bom: true)) == (text, "UTF-16LE", false))27 #expect(decoded(utf16(text, little: false, bom: true)) == (text, "UTF-16BE", false))28 // BOM-less UTF-16 is recognised from the position of its zero bytes.29 let ascii = "Plain ASCII text stored as UTF-16 without any byte-order mark.\n"30 #expect(decoded(utf16(ascii, little: true, bom: false)) == (ascii, "UTF-16LE", true))31 #expect(decoded(utf16(ascii, little: false, bom: false)) == (ascii, "UTF-16BE", true))32 // UTF-32 with BOM.33 var utf32 = Data([0xFF, 0xFE, 0x00, 0x00])34 for u in "A😀".unicodeScalars { withUnsafeBytes(of: u.value.littleEndian) { utf32.append(contentsOf: $0) } }35 #expect(decoded(utf32) == ("A😀", "UTF-32LE", false))36 // Legacy 8-bit text: Windows-1252, including its typographic characters in 0x80...0x9F.37 let cp1252 = Data([0x93, 0x51, 0x75, 0x6F, 0x74, 0x65, 0x94, 0x20, 0x63, 0x61, 0x66, 0xE9, 0x20, 0x80, 0x35])38 #expect(decoded(cp1252) == ("\u{201C}Quote\u{201D} café €5", "Windows-1252", true))39 // Bytes undefined in Windows-1252 fall back to ISO-8859-1 rather than failing.40 let latin1 = TextDecoder.decode(Data([0x41, 0x81, 0xE9, 0x8D]))41 #expect(latin1.guessed && latin1.text.hasPrefix("A") && latin1.text.contains("é") && latin1.text.unicodeScalars.count == 4)42 // Mostly valid UTF-8 with one damaged byte is repaired, not misread as an 8-bit encoding.43 var damaged = Data("Déjà vu, naïve façade, crème brûlée".utf8)44 damaged.append(0xFF)45 let repaired = TextDecoder.decode(damaged)46 #expect(repaired.encoding == "UTF-8" && repaired.replacedInvalidBytes && repaired.text == "Déjà vu, naïve façade, crème brûlée\u{FFFD}")47 // Odd-length and lone-surrogate UTF-16.48 let lone = TextDecoder.decode(Data([0xFF, 0xFE, 0x41, 0x00, 0x00, 0xD8, 0x42, 0x00, 0x43]))49 #expect(lone.text == "A\u{FFFD}B" && lone.replacedInvalidBytes)50 #expect(TextDecoder.decode(Data()).text == "")51 }5253 @Test func importSplitsLinesAndNormalisesEndings() throws {54 let result = try PlainText.read(Data("first\r\nsecond\rthird\n\nafter blank\twith tab\u{0C}new page\nnul\u{0}gone\u{FFFC}\n".utf8))55 #expect(Describe.blocks(result.document) == ["Normal: first", "Normal: second", "Normal: third", "Normal: ", "Normal: after blank\twith tab", "PAGEBREAK", "Normal: new page", "Normal: nulgone\u{FFFD}"])56 #expect(result.warnings.map(\.feature) == ["Control characters"])57 #expect(Describe.blocks(try PlainText.read(Data()).document) == ["Normal: "])58 #expect(Describe.blocks(try PlainText.read(Data("no newline".utf8)).document) == ["Normal: no newline"])59 #expect(Describe.blocks(try PlainText.read(Data("\n".utf8)).document) == ["Normal: "])60 #expect(Describe.blocks(try PlainText.read(Data("\n\n".utf8)).document) == ["Normal: ", "Normal: "])61 // A legacy encoding is reported to the user.62 let legacy = try PlainText.read(Data([0x63, 0x61, 0x66, 0xE9]))63 #expect(legacy.document.body[0].paragraph?.visibleText == "café")64 #expect(legacy.warnings.first?.severity == .approximated)65 #expect(legacy.warnings.first?.detail.contains("Windows-1252") == true)66 // No paragraph ever holds an inline object or formatting.67 #expect(result.document.body.compactMap(\.paragraph).allSatisfy { $0.runs.allSatisfy { $0.inline == nil && $0.props == .none } })68 }6970 @Test func roundTripIsExactForPlainDocuments() throws {71 let text = "Title\n\nFirst paragraph, with ünïcödé 中文 العربية 👩👩👧👦.\n\tIndented with a tab\n\n\nTwo blank lines above\u{0C}After a page break"72 let doc = try PlainText.read(Data(text.utf8)).document73 let exported = PlainText.export(doc)74 #expect(String(decoding: exported.data, as: UTF8.self) == text)75 #expect(exported.warnings.map(\.feature) == ["Encoding"])76 #expect(try PlainText.read(exported.data).document.body.count == doc.body.count)77 }7879 @Test func exportOfARichDocument() {80 let result = PlainText.export(Fixture.rich())81 let text = String(decoding: result.data, as: UTF8.self)82 let lines = text.components(separatedBy: "\n")83 #expect(lines[0] == "Rich fixture")84 #expect(lines[1...3] == ["Sommaire", "Introduction", " Second section"])85 #expect(lines.contains("Plain bold italic link styled inserted[A blue box][Photo]Notee^{i\\pi}+1=0[1][i][see @knuth1984, p. 4; -@doe:2020, passim]Figure 1?2026Rich « fixture »Ada Lovelace end"))86 #expect(lines.contains("second line\ttabbed"))87 #expect(lines.contains("Figure 1: A caption"))88 // Lists carry their computed labels, nested levels are indented.89 #expect(lines.contains("• First bullet") && lines.contains(" ◦ Nested bullet") && lines.contains("1. One") && lines.contains(" 1.1. One point one"))90 // Tables: one line per row, cells separated by tabs, merged-away cells empty.91 #expect(lines.contains("Header A+B\tTall") && lines.contains("a1\tb1\t"))92 // The page break and the odd-page section break are form feeds (the column break is not); notes are appended with their marks.93 #expect(text.contains("\na2\tline one line two\t42\u{0C}\u{0C}Second section\n"))94 #expect(text.hasSuffix("Références\n\n[1] A footnote with emphasis.\n[i] An endnote.\nSecond paragraph."))95 let features = Set(result.warnings.map(\.feature))96 for expected in ["Formatting", "Images", "Links", "Tables", "Equations", "Footnotes and endnotes", "Comments", "Tracked changes", "Generated lists", "Encoding"] {97 #expect(features.contains(expected), "missing warning: \(expected)")98 }99 #expect(result.warnings.first { $0.feature == "Formatting" }?.severity == .dropped)100 }101102 @Test func oversizedInputIsRefused() {103 let big = Data(count: FormatLimits.maxTextBytes + 1)104 #expect(throws: FormatError.self) { _ = try PlainText.read(big) }105 #expect(PlainText.importDocument(big).warnings.first?.feature == "Unreadable Plain Text")106 }107}108109@Suite struct PDFTextImportTests {110 @Test func textIsRebuiltIntoParagraphsWithPageBreaks() throws {111 let pdf = Fixture.pdf(pages: [112 ["Annual Report", "",113 "This is the first paragraph of the document, set on lines of about the same",114 "length, as a justified paragraph would be. A word is hyphen-",115 "ated across two lines here.",116 "The second paragraph follows without any blank line, but the previous line",117 "ended a sentence and was short.", "",118 "A third paragraph, separated by a blank line."],119 ["Second page heading", "", "Text on the second page."],120 ], title: "PDF Title")121 let result = try PDFTextImporter.read(pdf)122 let lines = Describe.blocks(result.document)123 #expect(lines.first == "Normal: Annual Report")124 #expect(lines.contains("PAGEBREAK"))125 #expect(lines.last == "Normal: Text on the second page.")126 let breakIndex = try #require(lines.firstIndex(of: "PAGEBREAK"))127 #expect(lines[breakIndex + 1] == "Normal: Second page heading")128 // Lines of one paragraph are joined; the hyphenated word is whole again.129 let all = Describe.texts(result.document)130 #expect(all.contains { $0.hasPrefix("This is the first paragraph") && $0.hasSuffix("A word is hyphenated across two lines here.") })131 #expect(all.contains { $0.hasPrefix("The second paragraph follows") && $0.hasSuffix("was short.") })132 #expect(all.contains("A third paragraph, separated by a blank line."))133 #expect(result.document.meta.title == "PDF Title")134 #expect(result.document.pageSetup.paper == .letter)135 // The loss of layout is always stated.136 let layout = try #require(result.warnings.first { $0.feature == "PDF layout" })137 #expect(layout.severity == .approximated)138 #expect(layout.detail.contains("layout") && layout.detail.contains("fonts") && layout.detail.contains("images") && layout.detail.contains("tables"))139 #expect(result.warnings.contains { $0.feature == "Hyphenation" && $0.detail.hasPrefix("1 word") })140 #expect(result.document.body.compactMap(\.paragraph).allSatisfy { $0.style == StyleID.normal && $0.runs.allSatisfy { $0.props == .none } })141 }142143 @Test func paragraphHeuristics() {144 var joined = 0145 #expect(PDFTextImporter.rebuildParagraphs("", joinedHyphens: &joined).isEmpty)146 #expect(PDFTextImporter.rebuildParagraphs(" \n\n \n", joinedHyphens: &joined).isEmpty)147 #expect(PDFTextImporter.rebuildParagraphs("one line", joinedHyphens: &joined) == ["one line"])148 #expect(PDFTextImporter.rebuildParagraphs("a first line that is quite long indeed and goes on\nand a second one of the same length, more or less\n\nnext", joinedHyphens: &joined)149 == ["a first line that is quite long indeed and goes on and a second one of the same length, more or less", "next"])150 // A compound word broken after its hyphen keeps the hyphen when the next line starts with a capital.151 #expect(PDFTextImporter.rebuildParagraphs("a long first line that happens to end with the compound Franco-\nGerman border, and then the long line that follows it here", joinedHyphens: &joined)152 == ["a long first line that happens to end with the compound Franco- German border, and then the long line that follows it here"])153 #expect(joined == 0)154 #expect(PDFTextImporter.rebuildParagraphs("inter-\nnational", joinedHyphens: &joined) == ["international"])155 #expect(joined == 1)156 }157158 @Test func passwordProtectedPDFIsRefusedClearly() {159 let locked = Fixture.pdf(pages: [["Secret text"]], password: "hunter2")160 do { _ = try PDFTextImporter.read(locked); Issue.record("a locked PDF must not be read") } catch let e as FormatError {161 guard case .unreadable(let why) = e else { Issue.record("\(e)"); return }162 #expect(why.contains("password"))163 } catch { Issue.record("\(error)") }164 let result = PDFTextImporter.importDocument(locked)165 #expect(result.warnings.count == 1 && result.warnings[0].severity == .dropped && result.warnings[0].detail.contains("password-protected"))166 #expect(Describe.texts(result.document) == [""]) // nothing leaked from the locked file167 }168169 @Test func scannedPDFWithoutTextLayer() throws {170 let scan = Fixture.pdf(pages: [[], []])171 do { _ = try PDFTextImporter.read(scan); Issue.record("a text-less PDF should be refused") } catch let e as FormatError {172 guard case .unreadable(let why) = e else { Issue.record("\(e)"); return }173 #expect(why.contains("no text layer") && why.contains("OCR"))174 } catch { Issue.record("\(error)") }175 // Some pages with text, some without: imported, with the number of empty pages reported.176 let mixed = try PDFTextImporter.read(Fixture.pdf(pages: [["Readable page."], [], ["Another readable page."]]))177 #expect(Describe.blocks(mixed.document) == ["Normal: Readable page.", "PAGEBREAK", "PAGEBREAK", "Normal: Another readable page."])178 #expect(mixed.warnings.contains { $0.feature == "Pages without text" && $0.severity == .dropped && $0.detail.hasPrefix("1 page has") })179 }180181 @Test func notAPDF() {182 #expect(throws: FormatError.unreadable("the file is not a PDF document")) { _ = try PDFTextImporter.read(Data("Hello".utf8)) }183 #expect(throws: FormatError.unreadable("the file is not a PDF document")) { _ = try PDFTextImporter.read(Data()) }184 #expect(throws: FormatError.self) { _ = try PDFTextImporter.read(Data("%PDF-1.7\ngarbage that is not a PDF body".utf8)) }185 let truncated = Fixture.pdf(pages: [["Some text"]]).prefix(200)186 #expect(throws: FormatError.self) { _ = try PDFTextImporter.read(Data(truncated)) }187 }188}189