Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocFormats/PlainText.swift3// Description: Plain text import (encoding detection, line-ending normalisation) and export (list labels, tab-separated tables,4// notes appended), plus the text decoder the Markdown and HTML importers use for raw bytes.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import DocModel10import DocContainer1112/// Decodes bytes of unknown encoding: BOM first, then strict UTF-8, BOM-less UTF-16, and finally Windows-1252 / ISO-8859-1.13public enum TextDecoder {14 public struct Decoded: Sendable {15 public var text: String16 /// "UTF-8", "UTF-16LE", "UTF-16BE", "UTF-32LE", "UTF-32BE", "Windows-1252", "ISO-8859-1".17 public var encoding: String18 /// True when the encoding was guessed rather than established (legacy 8-bit fallback, or repaired UTF-8).19 public var guessed: Bool20 public var replacedInvalidBytes: Bool21 }2223 public static func decode(_ data: Data) -> Decoded {24 let b = [UInt8](data.prefix(4))25 func body(_ skip: Int) -> Data { data.subdata(in: (data.startIndex + skip)..<data.endIndex) }26 if b.starts(with: [0xEF, 0xBB, 0xBF]) { return utf8(body(3), bom: true) }27 if b.starts(with: [0xFF, 0xFE, 0x00, 0x00]), data.count % 4 == 0 { return utf32(body(4), little: true) }28 if b.starts(with: [0x00, 0x00, 0xFE, 0xFF]) { return utf32(body(4), little: false) }29 if b.starts(with: [0xFF, 0xFE]) { return utf16(body(2), little: true, guessed: false) }30 if b.starts(with: [0xFE, 0xFF]) { return utf16(body(2), little: false, guessed: false) }31 if let s = String(data: data, encoding: .utf8) {32 // Text made of NULs in every other byte is BOM-less UTF-16, even though it is also valid UTF-8.33 if let little = looksLikeUTF16(data) { return utf16(data, little: little, guessed: true) }34 return Decoded(text: s, encoding: "UTF-8", guessed: false, replacedInvalidBytes: false)35 }36 if let little = looksLikeUTF16(data) { return utf16(data, little: little, guessed: true) }37 // Mostly valid UTF-8 with a few damaged bytes is repaired rather than misread as an 8-bit encoding.38 let stats = utf8Statistics(data)39 if stats.validMultibyte >= 2, stats.invalid * 3 <= stats.validMultibyte {40 return Decoded(text: String(decoding: data, as: UTF8.self), encoding: "UTF-8", guessed: true, replacedInvalidBytes: true)41 }42 if let s = String(data: data, encoding: .windowsCP1252) {43 return Decoded(text: s, encoding: "Windows-1252", guessed: true, replacedInvalidBytes: false)44 }45 return Decoded(text: String(data: data, encoding: .isoLatin1) ?? String(decoding: data, as: UTF8.self),46 encoding: "ISO-8859-1", guessed: true, replacedInvalidBytes: false)47 }4849 private static func utf8(_ d: Data, bom: Bool) -> Decoded {50 if let s = String(data: d, encoding: .utf8) { return Decoded(text: s, encoding: "UTF-8", guessed: false, replacedInvalidBytes: false) }51 return Decoded(text: String(decoding: d, as: UTF8.self), encoding: "UTF-8", guessed: false, replacedInvalidBytes: true)52 }5354 private static func utf16(_ d: Data, little: Bool, guessed: Bool) -> Decoded {55 var units: [UInt16] = []56 units.reserveCapacity(d.count / 2)57 let bytes = [UInt8](d)58 var i = 059 while i + 1 < bytes.count {60 units.append(little ? UInt16(bytes[i]) | UInt16(bytes[i + 1]) << 8 : UInt16(bytes[i]) << 8 | UInt16(bytes[i + 1]))61 i += 262 }63 var text = ""64 var repaired = bytes.count % 2 != 065 text.reserveCapacity(units.count)66 var decoder = UTF16()67 var iterator = units.makeIterator()68 loop: while true {69 switch decoder.decode(&iterator) {70 case .scalarValue(let u): text.unicodeScalars.append(u)71 case .emptyInput: break loop72 case .error: text.unicodeScalars.append("\u{FFFD}"); repaired = true73 }74 }75 return Decoded(text: text, encoding: little ? "UTF-16LE" : "UTF-16BE", guessed: guessed, replacedInvalidBytes: repaired)76 }7778 private static func utf32(_ d: Data, little: Bool) -> Decoded {79 let bytes = [UInt8](d)80 var text = ""81 var repaired = bytes.count % 4 != 082 var i = 083 while i + 3 < bytes.count {84 let v = little ? UInt32(bytes[i]) | UInt32(bytes[i + 1]) << 8 | UInt32(bytes[i + 2]) << 16 | UInt32(bytes[i + 3]) << 2485 : UInt32(bytes[i]) << 24 | UInt32(bytes[i + 1]) << 16 | UInt32(bytes[i + 2]) << 8 | UInt32(bytes[i + 3])86 if let u = Unicode.Scalar(v) { text.unicodeScalars.append(u) } else { text.unicodeScalars.append("\u{FFFD}"); repaired = true }87 i += 488 }89 return Decoded(text: text, encoding: little ? "UTF-32LE" : "UTF-32BE", guessed: false, replacedInvalidBytes: repaired)90 }9192 /// true = little endian, false = big endian, nil = not UTF-16. Looks at where the zero bytes of the first kilobytes fall.93 private static func looksLikeUTF16(_ data: Data) -> Bool? {94 let sample = [UInt8](data.prefix(4096))95 guard sample.count >= 4, data.count % 2 == 0 else { return nil }96 var evenZero = 0, oddZero = 097 for (i, byte) in sample.enumerated() where byte == 0 { if i % 2 == 0 { evenZero += 1 } else { oddZero += 1 } }98 let half = sample.count / 299 if oddZero * 10 >= half * 4, evenZero * 20 <= half { return true }100 if evenZero * 10 >= half * 4, oddZero * 20 <= half { return false }101 return nil102 }103104 private static func utf8Statistics(_ data: Data) -> (validMultibyte: Int, invalid: Int) {105 let b = [UInt8](data)106 var valid = 0, invalid = 0, i = 0107 while i < b.count {108 let c = b[i]109 if c < 0x80 { i += 1; continue }110 let need = c >= 0xF0 && c <= 0xF4 ? 3 : c >= 0xE0 ? 2 : c >= 0xC2 && c < 0xE0 ? 1 : -1111 if need > 0, (1...need).allSatisfy({ i + $0 < b.count && b[i + $0] & 0xC0 == 0x80 }) {112 valid += 1; i += need + 1113 } else {114 invalid += 1; i += 1115 }116 }117 return (valid, invalid)118 }119}120121public enum PlainText {122 /// Never throws: an oversized input yields an empty document and a `.dropped` warning. Use `read` to get the error instead.123 public static func importDocument(_ data: Data) -> ImportResult {124 do { return try read(data) } catch { return ImportFailure.result(error, format: .txt) }125 }126127 public static func read(_ data: Data) throws -> ImportResult {128 guard data.count <= FormatLimits.maxTextBytes else { throw FormatError.tooLarge("the text file is larger than \(FormatLimits.maxTextBytes) bytes") }129 var warnings = WarningCollector()130 let decoded = TextDecoder.decode(data)131 ImportFailure.reportEncoding(decoded, &warnings)132 var blocks: [Block] = []133 var removedControls = false134 // Split on LF, CR LF and CR; a form feed is a page break.135 var line = String.UnicodeScalarView()136 var previousCR = false137 func flush() {138 blocks.append(.paragraph(Paragraph(String(line))))139 line = String.UnicodeScalarView()140 }141 for u in decoded.text.unicodeScalars {142 switch u.value {143 case 0x0D: flush(); previousCR = true; continue144 case 0x0A: if !previousCR { flush() }145 case 0x0C:146 if !line.isEmpty { flush() }147 blocks.append(.pageBreak(BreakBlock()))148 case 0x09: line.append(u)149 case 0..<0x20, 0x7F: removedControls = true150 case 0xFFFC: line.append("\u{FFFD}"); removedControls = true151 default: line.append(u)152 }153 previousCR = false154 }155 // A final line terminator ends the last line; it does not open an extra empty paragraph.156 if !line.isEmpty || blocks.isEmpty { flush() }157 if removedControls { warnings.add(.dropped, "Control characters", "Control characters that cannot appear in a document were removed.") }158 let doc = DocumentModel(body: blocks)159 return ImportResult(document: doc, warnings: warnings.warnings)160 }161162 public static func export(_ doc: DocumentModel) -> ExportResult { export(doc, options: ExportOptions()) }163164 public static func export(_ source: DocumentModel, options: ExportOptions) -> ExportResult {165 var ctx = ExportContext(source, options: options)166 let doc = ctx.doc167 var lines: [String] = []168 func text(_ p: Paragraph) -> String {169 var t = ctx.flatText(p, noteMarks: true).replacingOccurrences(of: lineSeparatorCharacter, with: "\n")170 if ModelIdiom.isRule(p) { t = "----------------------------------------" }171 if let item = p.list, let label = ctx.numbering.listLabels[p.id] {172 t = String(repeating: " ", count: max(0, min(FormatLimits.maxListLevel, item.level))) + label + " " + t173 }174 return ctx.headingNumberPrefix(p) + t175 }176 for block in doc.body {177 switch block {178 case .paragraph(let p): lines.append(text(p))179 case .table(let t):180 for row in t.rows {181 lines.append(row.cells.map { cell in182 cell.vMerge == .continue ? "" : cell.paragraphs.map(text).joined(separator: " ").replacingOccurrences(of: "\n", with: " ").replacingOccurrences(of: "\t", with: " ")183 }.joined(separator: "\t"))184 }185 case .pageBreak(let b): if b.kind == .page { lines.append("\u{0C}") }186 case .sectionBreak(let s): if s.kind != .continuous { lines.append("\u{0C}") }187 case .toc(let toc):188 if let title = toc.title, !title.isEmpty { lines.append(title) }189 for e in ctx.entries(of: toc) { lines.append(String(repeating: " ", count: max(0, e.level - 1)) + e.text) }190 case .bibliography(let b):191 if let title = b.title, !title.isEmpty { lines.append(title) }192 if let entries = ctx.bibliographyParagraphs() { lines.append(contentsOf: entries.map(text)) }193 }194 }195 let notes = ctx.orderedNotes196 if !notes.isEmpty {197 lines.append("")198 for (ref, note) in notes {199 let body = note.paragraphs.map(text)200 lines.append("[\(ctx.noteMark(ref))] " + (body.first ?? ""))201 lines.append(contentsOf: body.dropFirst())202 }203 }204 // A form feed stands alone: it replaces the line break around it.205 var out = ""206 for (i, l) in lines.enumerated() {207 if l == "\u{0C}" { out += l; continue }208 if i > 0, lines[i - 1] != "\u{0C}" { out += "\n" }209 out += l210 }211 ctx.warnings.add(.info, "Encoding", "The text is written as UTF-8 without a byte-order mark.")212 return ExportResult(data: Data(out.utf8), warnings: ctx.finalWarnings(source: source, format: .txt))213 }214}215216/// Shared behaviour of the non-throwing `importDocument` entry points.217enum ImportFailure {218 static func result(_ error: Error, format: DocFormat) -> ImportResult {219 let why = (error as? FormatError)?.description ?? String(describing: error)220 return ImportResult(document: DocumentModel(), warnings: [FormatWarning(.dropped, feature: "Unreadable \(format.displayName)", detail: why)])221 }222223 static func reportEncoding(_ decoded: TextDecoder.Decoded, _ warnings: inout WarningCollector) {224 if decoded.replacedInvalidBytes {225 warnings.add(.approximated, "Text encoding", "The text contains bytes that are invalid in \(decoded.encoding); they were replaced by U+FFFD.")226 } else if decoded.guessed {227 warnings.add(.approximated, "Text encoding", "The text is not valid UTF-8 and declares no encoding; it was read as \(decoded.encoding). Check accented characters.")228 }229 }230}231