SPB Git

Repository Public

FormatSupport.swift 20.7 KB 413 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocFormats/FormatSupport.swift3// Description: Helpers shared by the importers: safety limits, text sanitising, image decoding (data URIs, type sniffing, size),4//              list allocation, link policy, model idioms (rules, display equations) and run-boundary healing.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import ImageIO10import CryptoKit11import DocModel12import DocContainer1314/// Safety limits applied by every importer of this module. Inputs beyond them are rejected with `FormatError.tooLarge`.15public enum FormatLimits {16    /// Markdown, HTML and plain text sources.17    public static let maxTextBytes = 64 * 1024 * 102418    public static let maxRTFBytes = 128 * 1024 * 102419    public static let maxPDFBytes = 512 * 1024 * 102420    public static let maxContainerBytes = 1024 * 1024 * 102421    /// One embedded image (data URI, resource loader result, RTF picture).22    public static let maxImageBytes = 64 * 1024 * 102423    /// Nested block containers (quotes, list items) the Markdown parser follows; deeper content is kept as text.24    public static let maxMarkdownNesting = 3225    /// Open elements the HTML tree builder tracks; deeper elements are treated as transparent.26    public static let maxHTMLDepth = 25627    /// Deepest list level of the model (zero-based).28    public static let maxListLevel = 829    public static let maxTableColumns = 25630    public static let maxTableCells = 1_000_00031    public static let maxPDFPages = 20_00032}3334/// Runs a closure synchronously on a thread with a large stack. Used by the recursive-descent Markdown parser so that its35/// (bounded) recursion never depends on the stack of the calling thread, which is 512 KB on secondary threads.36enum DeepStack {37    private final class Box<T>: @unchecked Sendable { var value: T? }3839    static func run<T>(stackSize: Int = 16 * 1024 * 1024, _ body: () -> T) -> T {40        withoutActuallyEscaping(body) { escaping in41            let result = Box<T>()42            let work = Box<() -> T>()43            work.value = escaping44            let done = DispatchSemaphore(value: 0)45            let thread = Thread {46                result.value = work.value?()47                work.value = nil48                done.signal()49            }50            thread.stackSize = stackSize51            thread.start()52            done.wait()53            return result.value!54        }55    }56}5758enum TextSanitizer {59    /// Makes external text safe for a run: C0 controls other than tab are removed, CR / LF / CRLF become U+2028 and U+FFFC60    /// (reserved for inline objects) becomes U+FFFD.61    static func clean(_ s: String) -> String {62        var needs = false63        for u in s.unicodeScalars where (u.value < 0x20 && u.value != 0x09) || u.value == 0xFFFC || u.value == 0x7F { needs = true; break }64        guard needs else { return s }65        var out = String.UnicodeScalarView()66        var previousCR = false67        for u in s.unicodeScalars {68            switch u.value {69            case 0x0D: out.append("\u{2028}"); previousCR = true; continue70            case 0x0A: if !previousCR { out.append("\u{2028}") }71            case 0x09: out.append(u)72            case 0..<0x20, 0x7F: break73            case 0xFFFC: out.append("\u{FFFD}")74            default: out.append(u)75            }76            previousCR = false77        }78        return String(out)79    }80}8182enum LinkPolicy {83    /// Returns the link target when it is safe to keep, nil for script-bearing schemes.84    static func sanitize(_ raw: String) -> String? {85        let url = raw.trimmingCharacters(in: .whitespacesAndNewlines)86        guard !url.isEmpty else { return nil }87        let compact = url.unicodeScalars.filter { $0.value > 0x20 }.map { String($0) }.joined().lowercased()88        for scheme in ["javascript:", "vbscript:", "data:", "livescript:"] where compact.hasPrefix(scheme) { return nil }89        return TextSanitizer.clean(url).replacingOccurrences(of: "\u{2028}", with: "")90    }91    static func isRemote(_ src: String) -> Bool {92        let s = src.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()93        if s.hasPrefix("//") { return true }94        guard let colon = s.firstIndex(of: ":") else { return false }95        let scheme = s[..<colon]96        guard !scheme.isEmpty, scheme.allSatisfy({ $0.isLetter || $0.isNumber || $0 == "+" || $0 == "-" || $0 == "." }) else { return false }97        // "c:" drive letters and file: URLs are local paths (refused by the image resolver), not remote resources.98        return scheme != "data" && scheme != "file" && scheme.count > 199    }100}101102enum ImageSupport {103    static func sniffMime(_ d: Data) -> String? {104        let b = [UInt8](d.prefix(16))105        guard b.count >= 12 else { return nil }106        if b.starts(with: [0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]) { return "image/png" }107        if b.starts(with: [0xFF, 0xD8, 0xFF]) { return "image/jpeg" }108        if b.starts(with: [0x47, 0x49, 0x46, 0x38]) { return "image/gif" }109        if b.starts(with: [0x49, 0x49, 0x2A, 0x00]) || b.starts(with: [0x4D, 0x4D, 0x00, 0x2A]) { return "image/tiff" }110        if b.starts(with: [0x25, 0x50, 0x44, 0x46]) { return "application/pdf" }111        if b.starts(with: [0x42, 0x4D]) { return "image/bmp" }112        if b.starts(with: [0x52, 0x49, 0x46, 0x46]), Array(b[8..<12]) == [0x57, 0x45, 0x42, 0x50] { return "image/webp" }113        if Array(b[4..<8]) == [0x66, 0x74, 0x79, 0x70] {114            let brand = String(decoding: b[8..<12], as: UTF8.self)115            if ["heic", "heix", "mif1", "msf1", "hevc"].contains(brand) { return "image/heic" }116        }117        return nil118    }119120    /// Natural size in points (pixels at the image's resolution; 96 dpi when it declares none), nil when ImageIO cannot read it.121    static func pointSize(_ d: Data) -> (width: Double, height: Double)? {122        guard let src = CGImageSourceCreateWithData(d as CFData, nil), CGImageSourceGetCount(src) > 0,123              let props = CGImageSourceCopyPropertiesAtIndex(src, 0, nil) as? [CFString: Any],124              let w = (props[kCGImagePropertyPixelWidth] as? NSNumber)?.doubleValue,125              let h = (props[kCGImagePropertyPixelHeight] as? NSNumber)?.doubleValue, w > 0, h > 0 else { return nil }126        let dx = (props[kCGImagePropertyDPIWidth] as? NSNumber)?.doubleValue ?? 96127        let dy = (props[kCGImagePropertyDPIHeight] as? NSNumber)?.doubleValue ?? dx128        return (w * 72 / max(1, dx), h * 72 / max(1, dy))129    }130131    static func pixelSize(_ d: Data) -> (width: Int, height: Int)? {132        guard let src = CGImageSourceCreateWithData(d as CFData, nil), CGImageSourceGetCount(src) > 0,133              let props = CGImageSourceCopyPropertiesAtIndex(src, 0, nil) as? [CFString: Any],134              let w = (props[kCGImagePropertyPixelWidth] as? NSNumber)?.intValue,135              let h = (props[kCGImagePropertyPixelHeight] as? NSNumber)?.intValue, w > 0, h > 0 else { return nil }136        return (w, h)137    }138139    /// Transcodes any image ImageIO reads to PNG (used by the RTF writer, which only carries PNG and JPEG).140    static func pngData(from d: Data) -> Data? {141        guard let src = CGImageSourceCreateWithData(d as CFData, nil), let image = CGImageSourceCreateImageAtIndex(src, 0, nil) else { return nil }142        let out = NSMutableData()143        guard let dest = CGImageDestinationCreateWithData(out, "public.png" as CFString, 1, nil) else { return nil }144        CGImageDestinationAddImage(dest, image, nil)145        return CGImageDestinationFinalize(dest) ? out as Data : nil146    }147148    /// Decodes a `data:` URI (base64 or percent-encoded). The declared media type is returned but never trusted.149    static func decodeDataURI(_ uri: String) -> Data? {150        let s = uri.trimmingCharacters(in: .whitespacesAndNewlines)151        guard s.count > 5, s.prefix(5).lowercased() == "data:", let comma = s.firstIndex(of: ",") else { return nil }152        let header = s[s.index(s.startIndex, offsetBy: 5)..<comma].lowercased()153        let payload = String(s[s.index(after: comma)...])154        if header.hasSuffix(";base64") {155            let compact = payload.removingPercentEncoding ?? payload156            guard let data = Data(base64Encoded: compact, options: .ignoreUnknownCharacters), !data.isEmpty else { return nil }157            return data158        }159        guard let decoded = payload.removingPercentEncoding else { return nil }160        return Data(decoded.utf8)161    }162163    static func dataURI(_ asset: Asset) -> String { "data:\(asset.mimeType);base64,\(asset.data.base64EncodedString())" }164}165166/// Resolves image sources met by the Markdown and HTML importers. Nothing is ever fetched: only `data:` URIs and the caller's167/// resource loader (for relative paths) produce images.168struct ImageResolver {169    private(set) var assets: [String: Asset] = [:]170    private var byDigest: [String: String] = [:]171    let loader: ((String) -> Data?)?172    /// Widest an imported image is allowed to be (the text width of a Letter page with one-inch margins).173    static let maxWidth: Double = 468174175    init(loader: ((String) -> Data?)?) { self.loader = loader }176177    /// Returns an inline image for `src`, or nil (with a warning) when the image cannot be embedded.178    mutating func resolve(src rawSrc: String, alt: String, warnings: inout WarningCollector) -> ImageObject? {179        let src = rawSrc.trimmingCharacters(in: .whitespacesAndNewlines)180        guard !src.isEmpty else { return nil }181        var data: Data?182        if src.lowercased().hasPrefix("data:") {183            data = ImageSupport.decodeDataURI(src)184            if data == nil { warnings.add(.dropped, "Images", "An embedded image (data URI) could not be decoded; its alternative text was kept."); return nil }185        } else if LinkPolicy.isRemote(src) {186            warnings.add(.dropped, "Remote images", "Remote images are never downloaded; the alternative text was kept.")187            return nil188        } else {189            var path = src190            if let cut = path.firstIndex(where: { $0 == "?" || $0 == "#" }) { path = String(path[..<cut]) }191            path = path.removingPercentEncoding ?? path192            let parts = path.split(separator: "/", omittingEmptySubsequences: false)193            if path.hasPrefix("/") || path.hasPrefix("~") || path.contains("\\") || path.contains("\0") || parts.contains("..") || path.contains(":") {194                warnings.add(.dropped, "Images", "An image path outside the document folder was refused; the alternative text was kept.")195                return nil196            }197            guard let loader else {198                warnings.add(.dropped, "Images", "Linked image files were not available; the alternative text was kept.")199                return nil200            }201            data = loader(path)202            if data == nil { warnings.add(.dropped, "Images", "A linked image file was not found; the alternative text was kept."); return nil }203        }204        guard let data else { return nil }205        return add(data, alt: alt, warnings: &warnings)206    }207208    mutating func add(_ data: Data, alt: String, warnings: inout WarningCollector) -> ImageObject? {209        guard data.count <= FormatLimits.maxImageBytes else {210            warnings.add(.dropped, "Images", "An image larger than the safety limit was skipped.")211            return nil212        }213        guard let mime = ImageSupport.sniffMime(data) else {214            warnings.add(.dropped, "Images", "An image in an unsupported format (for example SVG) was skipped; the alternative text was kept.")215            return nil216        }217        let digest = SHA256.hash(data: data).map { String(format: "%02x", $0) }.joined()218        let id: String219        if let known = byDigest[digest] { id = known } else {220            let asset = Asset(mimeType: mime, data: data)221            assets[asset.id] = asset222            byDigest[digest] = asset.id223            id = asset.id224        }225        var size = ImageSupport.pointSize(data) ?? (width: 240, height: 180)226        if size.width > ImageResolver.maxWidth {227            size = (ImageResolver.maxWidth, size.height * ImageResolver.maxWidth / size.width)228        }229        return ImageObject(asset: id, width: max(1, size.width), height: max(1, size.height), altText: TextSanitizer.clean(alt))230    }231}232233/// Hands out list definitions to importers. One definition per top-level list, so numbering restarts between lists;234/// a nested list whose flavour conflicts with an earlier sibling at the same level gets a definition of its own.235struct ListAllocator {236    private(set) var definitions: [ListDefinition] = []237    private var used: [String: [Int: Bool]] = [:]238239    private static func level(_ level: Int, ordered: Bool, start: Int, format: NumberFormat?) -> ListLevel {240        let template = ordered ? ListDefinition.numbered(id: "") : ListDefinition.bullets(id: "")241        var l = template.levels[max(0, min(template.levels.count - 1, level))]242        if ordered {243            // Markdown and HTML number nested lists with digits unless told otherwise.244            l.start = start245            l.format = format ?? .decimal246        }247        return l248    }249250    mutating func newList(ordered: Bool, start: Int = 1, format: NumberFormat? = nil) -> String {251        var def = ordered ? ListDefinition.numbered() : ListDefinition.bullets()252        def.levels[0] = ListAllocator.level(0, ordered: ordered, start: start, format: format)253        definitions.append(def)254        used[def.id] = [0: ordered]255        return def.id256    }257258    /// The list id to use for a list nested at `level` inside `parent`.259    mutating func sublist(parent: String, level: Int, ordered: Bool, start: Int = 1, format: NumberFormat? = nil) -> String {260        let lvl = max(0, min(FormatLimits.maxListLevel, level))261        guard let index = definitions.firstIndex(where: { $0.id == parent }) else { return newList(ordered: ordered, start: start, format: format) }262        let wanted = ListAllocator.level(lvl, ordered: ordered, start: start, format: format)263        if let flavour = used[parent]?[lvl] {264            if flavour == ordered && definitions[index].levels[lvl] == wanted { return parent }265            var copy = definitions[index]266            copy.id = DocID.make()267            copy.levels[lvl] = wanted268            definitions.append(copy)269            var marks = used[parent] ?? [:]270            marks[lvl] = ordered271            used[copy.id] = marks272            return copy.id273        }274        definitions[index].levels[lvl] = wanted275        used[parent, default: [:]][lvl] = ordered276        return parent277    }278}279280/// Prefix of the temporary note ids the importers use until footnote definitions are matched with their references.281let pendingNotePrefix = "\u{1}note:"282283/// Note references are met before (or without) their definitions. Importers first insert references with a temporary id284/// (`pendingNotePrefix` + target [+ U+0001 + visible mark]) and link them once every definition is known.285enum PendingNotes {286    static func parse(_ ref: NoteRef) -> (target: String, mark: String)? {287        guard ref.id.hasPrefix(pendingNotePrefix) else { return nil }288        let parts = ref.id.dropFirst(pendingNotePrefix.count).split(separator: "\u{1}", maxSplits: 1, omittingEmptySubsequences: false)289        return (String(parts.first ?? ""), parts.count > 1 ? String(parts[1]) : "")290    }291292    /// Definitions referenced from the body, directly or through other referenced notes.293    static func reachable(from body: [Block], definitions: [String: [Paragraph]]) -> Set<String> {294        var seen = Set<String>()295        var queue: [String] = []296        func scan(_ p: Paragraph) {297            for r in p.runs {298                if case .note(let ref)? = r.inline, let target = parse(ref)?.target, definitions[target] != nil, seen.insert(target).inserted { queue.append(target) }299            }300        }301        DocumentModel.forEachParagraph(in: body) { p, _ in scan(p) }302        while let target = queue.popLast() { definitions[target]?.forEach(scan) }303        return seen304    }305}306307/// Conventions used to express things the model has no dedicated type for.308enum ModelIdiom {309    /// A horizontal rule: an empty paragraph with a bottom border.310    static func rule() -> Paragraph {311        Paragraph(props: ParaProps(borderBottom: BorderLine(width: 0.75, color: RGBA(hex: 0x9AA0AE))))312    }313    static func isRule(_ p: Paragraph) -> Bool { p.isEmpty && p.props.borderBottom != nil && p.list == nil }314315    /// The equation of a display-equation paragraph (one display equation and nothing but white space around it).316    static func displayEquation(_ p: Paragraph) -> Equation? {317        var found: Equation?318        for r in p.runs {319            if let inline = r.inline {320                guard case .equation(let e) = inline, e.display, found == nil else { return nil }321                found = e322            } else if !r.text.allSatisfy({ $0.isWhitespace || $0 == "\u{2028}" }) {323                return nil324            }325        }326        return found327    }328329    /// 1...9 when the paragraph is a heading.330    static func headingLevel(_ p: Paragraph, styles: StyleSheet) -> Int? {331        if p.style.hasPrefix("Heading"), let n = Int(p.style.dropFirst(7)), (1...9).contains(n) { return n }332        let level = p.props.outlineLevel ?? styles.styleProps(p.style).para.outlineLevel ?? 0333        return (1...9).contains(level) ? level : nil334    }335336    /// Formatting a run carries on its own (character style chain + direct properties), without what the paragraph style contributes.337    static func explicitChar(_ r: Run, styles: StyleSheet) -> CharProps {338        var c = CharProps.none339        if let cs = r.charStyle { for s in styles.chain(cs) { c.apply(s.char) } }340        c.apply(r.props)341        return c342    }343344    static func isBold(_ c: CharProps) -> Bool { c.weight.map { $0 >= 600 } ?? (c.bold ?? false) }345}346347/// pandoc-compatible heading identifiers ("My Heading 2" → "my-heading-2"), unique within one document.348struct Slugger {349    private var seen: [String: Int] = [:]350    static func slug(_ text: String) -> String {351        var out = ""352        var pendingSpace = false353        for ch in text.lowercased() {354            if ch.isLetter || ch.isNumber || ch == "_" || ch == "-" || ch == "." {355                if pendingSpace, !out.isEmpty { out.append("-") }356                pendingSpace = false357                out.append(ch)358            } else if ch.isWhitespace { pendingSpace = true }359        }360        while let f = out.first, !f.isLetter { out.removeFirst() }361        return out.isEmpty ? "section" : out362    }363    mutating func unique(_ text: String) -> String {364        let base = Slugger.slug(text)365        let n = seen[base] ?? 0366        seen[base] = n + 1367        return n == 0 ? base : "\(base)-\(n)"368    }369}370371extension Paragraph {372    /// Merges equal neighbours, then moves any run boundary that falls inside a composed character sequence (a combining mark,373    /// a ZWJ emoji sequence, a surrogate pair) to the end of that sequence, so no run starts in the middle of a character.374    mutating func healRunBoundaries() {375        normalize()376        var i = 0377        while i + 1 < runs.count {378            let a = runs[i], b = runs[i + 1]379            // Two ASCII characters never combine (CR LF cannot occur here), which is the common case.380            let plainJunction = (a.text.utf8.last ?? 0) < 0x80 && (b.text.utf8.first ?? 0) < 0x80381            if !plainJunction, a.inline == nil, b.inline == nil, !a.text.isEmpty, !b.text.isEmpty {382                let aLen = a.text.utf16Length, bLen = b.text.utf16Length383                let tailStart = a.text.composedBoundary(before: aLen)384                let tail = a.text.utf16Substring(tailStart..<aLen)385                let headLen = b.text.composedBoundary(atOrBefore: min(bLen, 64))386                let joined = (tail + b.text.utf16Substring(0..<max(headLen, b.text.composedBoundary(after: 0)))) as NSString387                let tailLen = tail.utf16Length388                if tailLen < joined.length {389                    let r = joined.rangeOfComposedCharacterSequence(at: tailLen)390                    if r.location < tailLen {391                        let move = min(bLen, r.location + r.length - tailLen)392                        runs[i].text += b.text.utf16Substring(0..<move)393                        runs[i + 1].text = b.text.utf16Substring(move..<bLen)394                        if runs[i + 1].text.isEmpty { runs.remove(at: i + 1); continue }395                    }396                }397            }398            i += 1399        }400    }401}402403extension FormatError {404    /// Converts a ZIP-level failure into the module's error vocabulary.405    static func from(_ zip: ZipError) -> FormatError {406        switch zip {407        case .notAZipArchive: return .unreadable("the file is not an ORIX Docs container")408        case .tooManyEntries, .entryTooLarge, .archiveTooLarge, .suspiciousCompressionRatio: return .tooLarge(zip.description)409        case .corrupted, .unsupported, .unsafeEntryName, .checksumMismatch: return .integrityCheckFailed(zip.description)410        }411    }412}413
Files main
Menu

Theme