SPB Git

Repository Public

PlainText.swift 11.9 KB 231 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocFormats/PlainText.swift3// Description: Plain text import (encoding detection, line-ending normalisation) and export (list labels, tab-separated tables,4//              notes appended), plus the text decoder the Markdown and HTML importers use for raw bytes.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import DocModel10import DocContainer1112/// Decodes bytes of unknown encoding: BOM first, then strict UTF-8, BOM-less UTF-16, and finally Windows-1252 / ISO-8859-1.13public enum TextDecoder {14    public struct Decoded: Sendable {15        public var text: String16        /// "UTF-8", "UTF-16LE", "UTF-16BE", "UTF-32LE", "UTF-32BE", "Windows-1252", "ISO-8859-1".17        public var encoding: String18        /// True when the encoding was guessed rather than established (legacy 8-bit fallback, or repaired UTF-8).19        public var guessed: Bool20        public var replacedInvalidBytes: Bool21    }2223    public static func decode(_ data: Data) -> Decoded {24        let b = [UInt8](data.prefix(4))25        func body(_ skip: Int) -> Data { data.subdata(in: (data.startIndex + skip)..<data.endIndex) }26        if b.starts(with: [0xEF, 0xBB, 0xBF]) { return utf8(body(3), bom: true) }27        if b.starts(with: [0xFF, 0xFE, 0x00, 0x00]), data.count % 4 == 0 { return utf32(body(4), little: true) }28        if b.starts(with: [0x00, 0x00, 0xFE, 0xFF]) { return utf32(body(4), little: false) }29        if b.starts(with: [0xFF, 0xFE]) { return utf16(body(2), little: true, guessed: false) }30        if b.starts(with: [0xFE, 0xFF]) { return utf16(body(2), little: false, guessed: false) }31        if let s = String(data: data, encoding: .utf8) {32            // Text made of NULs in every other byte is BOM-less UTF-16, even though it is also valid UTF-8.33            if let little = looksLikeUTF16(data) { return utf16(data, little: little, guessed: true) }34            return Decoded(text: s, encoding: "UTF-8", guessed: false, replacedInvalidBytes: false)35        }36        if let little = looksLikeUTF16(data) { return utf16(data, little: little, guessed: true) }37        // Mostly valid UTF-8 with a few damaged bytes is repaired rather than misread as an 8-bit encoding.38        let stats = utf8Statistics(data)39        if stats.validMultibyte >= 2, stats.invalid * 3 <= stats.validMultibyte {40            return Decoded(text: String(decoding: data, as: UTF8.self), encoding: "UTF-8", guessed: true, replacedInvalidBytes: true)41        }42        if let s = String(data: data, encoding: .windowsCP1252) {43            return Decoded(text: s, encoding: "Windows-1252", guessed: true, replacedInvalidBytes: false)44        }45        return Decoded(text: String(data: data, encoding: .isoLatin1) ?? String(decoding: data, as: UTF8.self),46                       encoding: "ISO-8859-1", guessed: true, replacedInvalidBytes: false)47    }4849    private static func utf8(_ d: Data, bom: Bool) -> Decoded {50        if let s = String(data: d, encoding: .utf8) { return Decoded(text: s, encoding: "UTF-8", guessed: false, replacedInvalidBytes: false) }51        return Decoded(text: String(decoding: d, as: UTF8.self), encoding: "UTF-8", guessed: false, replacedInvalidBytes: true)52    }5354    private static func utf16(_ d: Data, little: Bool, guessed: Bool) -> Decoded {55        var units: [UInt16] = []56        units.reserveCapacity(d.count / 2)57        let bytes = [UInt8](d)58        var i = 059        while i + 1 < bytes.count {60            units.append(little ? UInt16(bytes[i]) | UInt16(bytes[i + 1]) << 8 : UInt16(bytes[i]) << 8 | UInt16(bytes[i + 1]))61            i += 262        }63        var text = ""64        var repaired = bytes.count % 2 != 065        text.reserveCapacity(units.count)66        var decoder = UTF16()67        var iterator = units.makeIterator()68        loop: while true {69            switch decoder.decode(&iterator) {70            case .scalarValue(let u): text.unicodeScalars.append(u)71            case .emptyInput: break loop72            case .error: text.unicodeScalars.append("\u{FFFD}"); repaired = true73            }74        }75        return Decoded(text: text, encoding: little ? "UTF-16LE" : "UTF-16BE", guessed: guessed, replacedInvalidBytes: repaired)76    }7778    private static func utf32(_ d: Data, little: Bool) -> Decoded {79        let bytes = [UInt8](d)80        var text = ""81        var repaired = bytes.count % 4 != 082        var i = 083        while i + 3 < bytes.count {84            let v = little ? UInt32(bytes[i]) | UInt32(bytes[i + 1]) << 8 | UInt32(bytes[i + 2]) << 16 | UInt32(bytes[i + 3]) << 2485                           : UInt32(bytes[i]) << 24 | UInt32(bytes[i + 1]) << 16 | UInt32(bytes[i + 2]) << 8 | UInt32(bytes[i + 3])86            if let u = Unicode.Scalar(v) { text.unicodeScalars.append(u) } else { text.unicodeScalars.append("\u{FFFD}"); repaired = true }87            i += 488        }89        return Decoded(text: text, encoding: little ? "UTF-32LE" : "UTF-32BE", guessed: false, replacedInvalidBytes: repaired)90    }9192    /// true = little endian, false = big endian, nil = not UTF-16. Looks at where the zero bytes of the first kilobytes fall.93    private static func looksLikeUTF16(_ data: Data) -> Bool? {94        let sample = [UInt8](data.prefix(4096))95        guard sample.count >= 4, data.count % 2 == 0 else { return nil }96        var evenZero = 0, oddZero = 097        for (i, byte) in sample.enumerated() where byte == 0 { if i % 2 == 0 { evenZero += 1 } else { oddZero += 1 } }98        let half = sample.count / 299        if oddZero * 10 >= half * 4, evenZero * 20 <= half { return true }100        if evenZero * 10 >= half * 4, oddZero * 20 <= half { return false }101        return nil102    }103104    private static func utf8Statistics(_ data: Data) -> (validMultibyte: Int, invalid: Int) {105        let b = [UInt8](data)106        var valid = 0, invalid = 0, i = 0107        while i < b.count {108            let c = b[i]109            if c < 0x80 { i += 1; continue }110            let need = c >= 0xF0 && c <= 0xF4 ? 3 : c >= 0xE0 ? 2 : c >= 0xC2 && c < 0xE0 ? 1 : -1111            if need > 0, (1...need).allSatisfy({ i + $0 < b.count && b[i + $0] & 0xC0 == 0x80 }) {112                valid += 1; i += need + 1113            } else {114                invalid += 1; i += 1115            }116        }117        return (valid, invalid)118    }119}120121public enum PlainText {122    /// Never throws: an oversized input yields an empty document and a `.dropped` warning. Use `read` to get the error instead.123    public static func importDocument(_ data: Data) -> ImportResult {124        do { return try read(data) } catch { return ImportFailure.result(error, format: .txt) }125    }126127    public static func read(_ data: Data) throws -> ImportResult {128        guard data.count <= FormatLimits.maxTextBytes else { throw FormatError.tooLarge("the text file is larger than \(FormatLimits.maxTextBytes) bytes") }129        var warnings = WarningCollector()130        let decoded = TextDecoder.decode(data)131        ImportFailure.reportEncoding(decoded, &warnings)132        var blocks: [Block] = []133        var removedControls = false134        // Split on LF, CR LF and CR; a form feed is a page break.135        var line = String.UnicodeScalarView()136        var previousCR = false137        func flush() {138            blocks.append(.paragraph(Paragraph(String(line))))139            line = String.UnicodeScalarView()140        }141        for u in decoded.text.unicodeScalars {142            switch u.value {143            case 0x0D: flush(); previousCR = true; continue144            case 0x0A: if !previousCR { flush() }145            case 0x0C:146                if !line.isEmpty { flush() }147                blocks.append(.pageBreak(BreakBlock()))148            case 0x09: line.append(u)149            case 0..<0x20, 0x7F: removedControls = true150            case 0xFFFC: line.append("\u{FFFD}"); removedControls = true151            default: line.append(u)152            }153            previousCR = false154        }155        // A final line terminator ends the last line; it does not open an extra empty paragraph.156        if !line.isEmpty || blocks.isEmpty { flush() }157        if removedControls { warnings.add(.dropped, "Control characters", "Control characters that cannot appear in a document were removed.") }158        let doc = DocumentModel(body: blocks)159        return ImportResult(document: doc, warnings: warnings.warnings)160    }161162    public static func export(_ doc: DocumentModel) -> ExportResult { export(doc, options: ExportOptions()) }163164    public static func export(_ source: DocumentModel, options: ExportOptions) -> ExportResult {165        var ctx = ExportContext(source, options: options)166        let doc = ctx.doc167        var lines: [String] = []168        func text(_ p: Paragraph) -> String {169            var t = ctx.flatText(p, noteMarks: true).replacingOccurrences(of: lineSeparatorCharacter, with: "\n")170            if ModelIdiom.isRule(p) { t = "----------------------------------------" }171            if let item = p.list, let label = ctx.numbering.listLabels[p.id] {172                t = String(repeating: "    ", count: max(0, min(FormatLimits.maxListLevel, item.level))) + label + " " + t173            }174            return ctx.headingNumberPrefix(p) + t175        }176        for block in doc.body {177            switch block {178            case .paragraph(let p): lines.append(text(p))179            case .table(let t):180                for row in t.rows {181                    lines.append(row.cells.map { cell in182                        cell.vMerge == .continue ? "" : cell.paragraphs.map(text).joined(separator: " ").replacingOccurrences(of: "\n", with: " ").replacingOccurrences(of: "\t", with: " ")183                    }.joined(separator: "\t"))184                }185            case .pageBreak(let b): if b.kind == .page { lines.append("\u{0C}") }186            case .sectionBreak(let s): if s.kind != .continuous { lines.append("\u{0C}") }187            case .toc(let toc):188                if let title = toc.title, !title.isEmpty { lines.append(title) }189                for e in ctx.entries(of: toc) { lines.append(String(repeating: "    ", count: max(0, e.level - 1)) + e.text) }190            case .bibliography(let b):191                if let title = b.title, !title.isEmpty { lines.append(title) }192                if let entries = ctx.bibliographyParagraphs() { lines.append(contentsOf: entries.map(text)) }193            }194        }195        let notes = ctx.orderedNotes196        if !notes.isEmpty {197            lines.append("")198            for (ref, note) in notes {199                let body = note.paragraphs.map(text)200                lines.append("[\(ctx.noteMark(ref))] " + (body.first ?? ""))201                lines.append(contentsOf: body.dropFirst())202            }203        }204        // A form feed stands alone: it replaces the line break around it.205        var out = ""206        for (i, l) in lines.enumerated() {207            if l == "\u{0C}" { out += l; continue }208            if i > 0, lines[i - 1] != "\u{0C}" { out += "\n" }209            out += l210        }211        ctx.warnings.add(.info, "Encoding", "The text is written as UTF-8 without a byte-order mark.")212        return ExportResult(data: Data(out.utf8), warnings: ctx.finalWarnings(source: source, format: .txt))213    }214}215216/// Shared behaviour of the non-throwing `importDocument` entry points.217enum ImportFailure {218    static func result(_ error: Error, format: DocFormat) -> ImportResult {219        let why = (error as? FormatError)?.description ?? String(describing: error)220        return ImportResult(document: DocumentModel(), warnings: [FormatWarning(.dropped, feature: "Unreadable \(format.displayName)", detail: why)])221    }222223    static func reportEncoding(_ decoded: TextDecoder.Decoded, _ warnings: inout WarningCollector) {224        if decoded.replacedInvalidBytes {225            warnings.add(.approximated, "Text encoding", "The text contains bytes that are invalid in \(decoded.encoding); they were replaced by U+FFFD.")226        } else if decoded.guessed {227            warnings.add(.approximated, "Text encoding", "The text is not valid UTF-8 and declares no encoding; it was read as \(decoded.encoding). Check accented characters.")228        }229    }230}231
Files main
Menu

Theme