SPB Git

Repository Public

MarkdownImporter.swift 39.3 KB 802 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocFormats/MarkdownImporter.swift3// Description: Markdown import: CommonMark core blocks, GFM tables / strikethrough / task lists, footnotes, YAML front matter,4//              LaTeX math, pandoc citations. A hand-written parser with bounded nesting; nothing is fetched or executed.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import DocModel10import DocContainer1112public enum MarkdownImporter {13    /// Never throws: an oversized input yields an empty document and a `.dropped` warning. Use `read` to get the error instead.14    /// `resourceLoader` receives the relative path of a linked image and returns its bytes; remote URLs are never fetched.15    public static func importDocument(_ text: String, resourceLoader: ((String) -> Data?)? = nil) -> ImportResult {16        do { return try read(text, resourceLoader: resourceLoader) } catch { return ImportFailure.result(error, format: .markdown) }17    }1819    public static func read(_ text: String, resourceLoader: ((String) -> Data?)? = nil) throws -> ImportResult {20        guard text.utf8.count <= FormatLimits.maxTextBytes else { throw FormatError.tooLarge("the Markdown source is larger than \(FormatLimits.maxTextBytes) bytes") }21        // The block parser is recursive (bounded by FormatLimits.maxMarkdownNesting); it runs on a stack sized for that bound,22        // whatever the size of the caller's stack.23        return DeepStack.run {24            let parser = MarkdownParser(loader: resourceLoader)25            return parser.run(text.hasPrefix("\u{FEFF}") ? String(text.dropFirst()) : text)26        }27    }28}2930final class MarkdownParser {31    struct Context {32        var depth = 033        var quote = 034        var list: (id: String, level: Int)?35        var inNote = false36    }3738    var warnings = WarningCollector()39    var images: ImageResolver40    var lists = ListAllocator()41    var references: [String: String] = [:]42    var noteLabels: Set<String> = []43    private var noteBodies: [String: [Paragraph]] = [:]44    private var meta = DocumentMetadata()4546    init(loader: ((String) -> Data?)?) { images = ImageResolver(loader: loader) }4748    static func normalizeLabel(_ s: String) -> String {49        s.lowercased().split(whereSeparator: { $0.isWhitespace }).joined(separator: " ")50    }5152    // MARK: Entry point5354    func run(_ source: String) -> ImportResult {55        var lines = splitLines(source)56        lines = frontMatter(lines)57        prescan(lines)58        var body = parseBlocks(lines, Context())59        var notes: [String: Note] = [:]60        resolveNotes(&body, &notes)61        linkHeadingAnchors(&body)62        var doc = DocumentModel(meta: meta, body: body)63        doc.lists = lists.definitions64        doc.notes = notes65        doc.assets = images.assets66        return ImportResult(document: doc, warnings: warnings.warnings)67    }6869    /// Splits into lines, normalising line endings and removing characters the model cannot hold.70    private func splitLines(_ source: String) -> [String] {71        var lines: [String] = []72        var line = String.UnicodeScalarView()73        var previousCR = false74        var removed = false75        for u in source.unicodeScalars {76            switch u.value {77            case 0x0D: lines.append(String(line)); line = String.UnicodeScalarView(); previousCR = true; continue78            case 0x0A: if !previousCR { lines.append(String(line)); line = String.UnicodeScalarView() }79            case 0x09: line.append(u)80            case 0..<0x20, 0x7F: removed = true81            case 0xFFFC: line.append("\u{FFFD}")82            case 0x2028, 0x2029: lines.append(String(line)); line = String.UnicodeScalarView()83            default: line.append(u)84            }85            previousCR = false86        }87        lines.append(String(line))88        if removed { warnings.add(.dropped, "Control characters", "Control characters that cannot appear in a document were removed.") }89        return lines90    }9192    // MARK: Line helpers9394    private func isBlank(_ l: String) -> Bool { l.utf8.allSatisfy { $0 == 0x20 || $0 == 0x09 } }9596    /// Width of the leading white space in columns (tab stops every four columns).97    private func indent(_ l: String) -> Int {98        var col = 099        for b in l.utf8 {100            if b == 0x20 { col += 1 } else if b == 0x09 { col += 4 - col % 4 } else { break }101        }102        return col103    }104105    /// Removes up to `columns` columns of leading white space.106    private func strip(_ l: String, columns: Int) -> String {107        var col = 0108        var index = l.startIndex109        while index < l.endIndex, col < columns {110            let ch = l[index]111            if ch == " " { col += 1 } else if ch == "\t" {112                let width = 4 - col % 4113                if col + width > columns { return String(repeating: " ", count: col + width - columns) + l[l.index(after: index)...] }114                col += width115            } else { break }116            index = l.index(after: index)117        }118        return String(l[index...])119    }120121    private func trimLeading(_ l: String) -> String { String(l.drop { $0 == " " || $0 == "\t" }) }122    private func trimmed(_ l: String) -> String { l.trimmingCharacters(in: CharacterSet(charactersIn: " \t")) }123124    private func atxHeading(_ l: String) -> (level: Int, text: String)? {125        let t = trimLeading(l)126        var level = 0127        for ch in t { if ch == "#" { level += 1 } else { break } }128        guard (1...6).contains(level) else { return nil }129        let rest = t.dropFirst(level)130        guard rest.isEmpty || rest.first == " " || rest.first == "\t" else { return nil }131        var text = trimmed(String(rest))132        // Optional closing sequence of #.133        if let lastSpace = text.lastIndex(of: " "), text[text.index(after: lastSpace)...].allSatisfy({ $0 == "#" }), text.last == "#" {134            text = trimmed(String(text[..<lastSpace]))135        } else if !text.isEmpty, text.allSatisfy({ $0 == "#" }) { text = "" }136        return (level, text)137    }138139    private func isThematicBreak(_ l: String) -> Bool {140        var marker: Character?141        var count = 0142        for ch in l {143            if ch == " " || ch == "\t" { continue }144            guard ch == "-" || ch == "*" || ch == "_" else { return false }145            if let m = marker, m != ch { return false }146            marker = ch; count += 1147        }148        return count >= 3149    }150151    private func setextLevel(_ l: String) -> Int? {152        let t = trimmed(l)153        guard !t.isEmpty, indent(l) < 4 else { return nil }154        if t.allSatisfy({ $0 == "=" }) { return 1 }155        if t.allSatisfy({ $0 == "-" }) { return 2 }156        return nil157    }158159    private func fenceStart(_ l: String) -> (char: Character, count: Int, indent: Int, info: String)? {160        let t = trimLeading(l)161        guard let c = t.first, c == "`" || c == "~" else { return nil }162        let count = t.prefix { $0 == c }.count163        guard count >= 3 else { return nil }164        let info = trimmed(String(t.dropFirst(count)))165        if c == "`", info.contains("`") { return nil }166        return (c, count, indent(l), info)167    }168169    private func quoteStrip(_ l: String) -> String? {170        guard indent(l) < 4 else { return nil }171        let t = trimLeading(l)172        guard t.first == ">" else { return nil }173        let rest = t.dropFirst()174        return rest.first == " " ? String(rest.dropFirst()) : String(rest)175    }176177    struct Marker {178        var ordered: Bool179        var number: Int180        var delimiter: Character181        var contentOffset: Int182        var rest: String183    }184185    private func listMarker(_ l: String) -> Marker? {186        let ind = indent(l)187        guard ind < 4 else { return nil }188        let t = trimLeading(l)189        guard let first = t.first else { return nil }190        var markerLength = 0191        var ordered = false, number = 1192        var delimiter = first193        if first == "-" || first == "+" || first == "*" {194            markerLength = 1195        } else if first.isASCII, first.isNumber {196            let digits = t.prefix { $0.isASCII && $0.isNumber }197            guard digits.count <= 9, let n = Int(digits) else { return nil }198            let after = t.dropFirst(digits.count)199            guard let d = after.first, d == "." || d == ")" else { return nil }200            ordered = true; number = n; delimiter = d; markerLength = digits.count + 1201        } else { return nil }202        let after = String(t.dropFirst(markerLength))203        guard after.isEmpty || after.first == " " || after.first == "\t" else { return nil }204        if isBlank(after) { return Marker(ordered: ordered, number: number, delimiter: delimiter, contentOffset: ind + markerLength + 1, rest: "") }205        let spaces = indent(after)206        if spaces >= 5 {207            return Marker(ordered: ordered, number: number, delimiter: delimiter, contentOffset: ind + markerLength + 1, rest: strip(after, columns: 1))208        }209        return Marker(ordered: ordered, number: number, delimiter: delimiter, contentOffset: ind + markerLength + spaces, rest: trimLeading(after))210    }211212    private func sameList(_ a: Marker, _ b: Marker) -> Bool { a.ordered == b.ordered && a.delimiter == b.delimiter }213214    private func tableDelimiter(_ l: String) -> [TextAlignment?]? {215        guard l.contains("-"), indent(l) < 4 else { return nil }216        let cells = splitRow(l)217        guard !cells.isEmpty, cells.count <= FormatLimits.maxTableColumns else { return nil }218        var out: [TextAlignment?] = []219        for cell in cells {220            let c = trimmed(cell)221            let left = c.hasPrefix(":"), right = c.hasSuffix(":") && c.count > 1222            let dashes = c.dropFirst(left ? 1 : 0).dropLast(right ? 1 : 0)223            guard !dashes.isEmpty, dashes.allSatisfy({ $0 == "-" }) else { return nil }224            out.append(left && right ? .center : right ? .right : left ? .left : nil)225        }226        return out227    }228229    private func splitRow(_ l: String) -> [String] {230        var t = trimmed(l)231        if t.hasPrefix("|") { t.removeFirst() }232        var cells: [String] = []233        var cell = ""234        var escaped = false235        for ch in t {236            if escaped { cell.append(ch); escaped = false; continue }237            if ch == "\\" { cell.append(ch); escaped = true; continue }238            if ch == "|" { cells.append(cell); cell = ""; continue }239            cell.append(ch)240        }241        if !isBlank(cell) || cells.isEmpty { cells.append(cell) }242        return cells243    }244245    /// The dashed line of a pandoc simple table ("------ -------"): the character ranges of its columns.246    private func simpleTableRule(_ l: String) -> [Range<Int>]? {247        guard indent(l) < 4 else { return nil }248        var groups: [Range<Int>] = []249        var start: Int?250        var column = 0251        for ch in l {252            if ch == "-" { if start == nil { start = column } }253            else if ch == " " { if let s = start { groups.append(s..<column); start = nil } }254            else { return nil }255            column += 1256        }257        if let s = start { groups.append(s..<column) }258        guard groups.count >= 2, groups.count <= FormatLimits.maxTableColumns, groups.allSatisfy({ $0.count >= 2 }) else { return nil }259        return groups260    }261262    /// A pandoc simple table starting at `lines[i]` (its header line): header, dashed rule, rows up to a blank line.263    private func simpleTable(_ lines: [String], at i: Int, _ ctx: Context) -> (table: Table, next: Int)? {264        guard i + 1 < lines.count, !isBlank(lines[i]), indent(lines[i]) < 4, let columns = simpleTableRule(lines[i + 1]) else { return nil }265        func cells(_ line: String) -> [String] {266            let chars = Array(line)267            return columns.enumerated().map { k, range in268                let lower = k == 0 ? 0 : range.lowerBound269                let upper = k + 1 < columns.count ? columns[k + 1].lowerBound : chars.count270                guard lower < chars.count else { return "" }271                return trimmed(String(chars[lower..<min(chars.count, max(lower, upper))]))272            }273        }274        // Alignment follows the position of the header text over its dashes.275        let header = Array(lines[i])276        var alignments: [TextAlignment?] = []277        for range in columns {278            let slice = range.lowerBound < header.count ? Array(header[range.lowerBound..<min(header.count, range.upperBound)]) : []279            let leftFlush = slice.first.map { $0 != " " } ?? false280            let rightFlush = slice.count == range.count && (slice.last.map { $0 != " " } ?? false)281            alignments.append(leftFlush && rightFlush ? nil : rightFlush ? .right : leftFlush ? .left : slice.contains { $0 != " " } ? .center : nil)282        }283        var rows = [cells(lines[i])]284        var j = i + 2285        while j < lines.count, !isBlank(lines[j]), rows.count * columns.count < FormatLimits.maxTableCells {286            if simpleTableRule(lines[j]) != nil || trimmed(lines[j]).allSatisfy({ $0 == "-" }) { j += 1; break }      // closing rule287            rows.append(cells(lines[j])); j += 1288        }289        return (table(rows, alignments, ctx), j)290    }291292    private func linkReference(_ l: String) -> (label: String, url: String)? {293        guard indent(l) < 4 else { return nil }294        let t = trimLeading(l)295        guard t.hasPrefix("["), !t.hasPrefix("[^"), let close = t.range(of: "]:") else { return nil }296        let label = String(t[t.index(after: t.startIndex)..<close.lowerBound])297        guard !label.isEmpty, !label.contains("["), !label.contains("]") else { return nil }298        var rest = trimmed(String(t[close.upperBound...]))299        guard !rest.isEmpty else { return nil }300        var url: String301        if rest.hasPrefix("<"), let end = rest.firstIndex(of: ">") {302            url = String(rest[rest.index(after: rest.startIndex)..<end]); rest = trimmed(String(rest[rest.index(after: end)...]))303        } else {304            url = String(rest.prefix { $0 != " " && $0 != "\t" }); rest = trimmed(String(rest.dropFirst(url.count)))305        }306        if !rest.isEmpty {307            guard let q = rest.first, let e = rest.last, (q == "\"" && e == "\"") || (q == "'" && e == "'") || (q == "(" && e == ")"), rest.count >= 2 else { return nil }308        }309        return (label, url)310    }311312    private func footnoteDefinition(_ l: String) -> (label: String, rest: String)? {313        guard indent(l) < 4 else { return nil }314        let t = trimLeading(l)315        guard t.hasPrefix("[^"), let close = t.range(of: "]:") else { return nil }316        let label = String(t[t.index(t.startIndex, offsetBy: 2)..<close.lowerBound])317        guard !label.isEmpty, !label.contains(where: { $0.isWhitespace || $0 == "[" || $0 == "]" }) else { return nil }318        return (label, trimLeading(String(t[close.upperBound...])))319    }320321    private func mathBlock(_ lines: [String], at i: Int) -> (latex: String, next: Int)? {322        let t = trimmed(lines[i])323        guard t.hasPrefix("$$"), indent(lines[i]) < 4 else { return nil }324        let rest = String(t.dropFirst(2))325        if rest.contains("$$") {326            guard rest.hasSuffix("$$") else { return nil }327            let latex = trimmed(String(rest.dropLast(2)))328            return latex.isEmpty ? nil : (latex, i + 1)329        }330        var body = [rest]331        var j = i + 1332        while j < lines.count, j - i < 2000 {333            if let end = lines[j].range(of: "$$") {334                guard isBlank(String(lines[j][end.upperBound...])) else { return nil }335                body.append(String(lines[j][..<end.lowerBound]))336                let latex = body.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines)337                return latex.isEmpty ? nil : (latex, j + 1)338            }339            body.append(lines[j]); j += 1340        }341        return nil342    }343344    /// True when `lines[i]` starts a block that interrupts a paragraph.345    private func startsBlock(_ lines: [String], at i: Int) -> Bool {346        let l = lines[i]347        if indent(l) >= 4 { return false }348        if i + 1 < lines.count, !isBlank(l), simpleTableRule(lines[i + 1]) != nil, simpleTableRule(l) == nil { return true }      // header line of a pandoc simple table349        if atxHeading(l) != nil || fenceStart(l) != nil || quoteStrip(l) != nil || isThematicBreak(l) { return true }350        if let m = listMarker(l), !m.rest.isEmpty, !m.ordered || m.number == 1 { return true }351        let t = trimLeading(l)352        if t.hasPrefix("$$") || t.hasPrefix(":::") { return true }353        if l.contains("|"), i + 1 < lines.count, let d = tableDelimiter(lines[i + 1]), d.count == splitRow(l).count { return true }354        return false355    }356357    // MARK: Front matter and pre-scan358359    private func frontMatter(_ lines: [String]) -> [String] {360        guard let first = lines.first, trimmed(first) == "---", lines.count > 2 else { return lines }361        guard let end = lines.dropFirst().firstIndex(where: { let t = trimmed($0); return t == "---" || t == "..." }), end > 1 || lines.count > 2 else { return lines }362        let yaml = Array(lines[1..<end])363        // A "---" paragraph break followed by text and another rule is not front matter: require key: value on the first content line.364        guard let firstContent = yaml.first(where: { !isBlank($0) }), firstContent.contains(":"), !firstContent.hasPrefix(" ") else { return lines }365        var ignored: [String] = []366        var i = 0367        func unquote(_ v: String) -> String {368            let t = trimmed(v)369            if t.count >= 2, let f = t.first, let l = t.last, (f == "\"" && l == "\"") || (f == "'" && l == "'") {370                let inner = String(t.dropFirst().dropLast())371                return f == "\"" ? inner.replacingOccurrences(of: "\\\"", with: "\"").replacingOccurrences(of: "\\\\", with: "\\") : inner.replacingOccurrences(of: "''", with: "'")372            }373            return t374        }375        while i < yaml.count {376            let line = yaml[i]377            i += 1378            guard !isBlank(line), !line.hasPrefix("#"), let colon = line.firstIndex(of: ":"), !line.hasPrefix(" ") else { continue }379            let key = trimmed(String(line[..<colon])).lowercased()380            var value = trimmed(String(line[line.index(after: colon)...]))381            var items: [String] = []382            if value.hasPrefix("["), value.hasSuffix("]") {383                items = value.dropFirst().dropLast().split(separator: ",").map { unquote(String($0)) }.filter { !$0.isEmpty }384                value = items.joined(separator: ", ")385            } else if value.isEmpty || value == "|" || value == ">" || value == "|-" || value == ">-" {386                var block: [String] = []387                while i < yaml.count, yaml[i].hasPrefix(" ") || yaml[i].hasPrefix("-") || isBlank(yaml[i]) {388                    let t = trimmed(yaml[i])389                    if t.hasPrefix("- ") { items.append(unquote(String(t.dropFirst(2)))) } else if !t.isEmpty { block.append(t) }390                    i += 1391                }392                value = items.isEmpty ? block.joined(separator: " ") : items.joined(separator: ", ")393            } else {394                value = unquote(value)395                items = [value]396            }397            value = TextSanitizer.clean(value)398            switch key {399            case "title": meta.title = value400            case "author", "authors": meta.author = value401            case "keywords", "tags": meta.keywords = items.isEmpty ? value.split(separator: ",").map { trimmed(String($0)) }.filter { !$0.isEmpty } : items.count == 1 ? items[0].split(separator: ",").map { trimmed(String($0)) }.filter { !$0.isEmpty } : items402            case "lang", "language": if !value.isEmpty { meta.language = value }403            case "subject", "description": meta.subject = value404            default: ignored.append(key)405            }406        }407        if !ignored.isEmpty {408            warnings.add(.dropped, "Front matter", "Front matter keys other than title, author, keywords, lang and subject are not kept: " + ignored.prefix(8).joined(separator: ", ") + ".")409        }410        return Array(lines[(end + 1)...])411    }412413    /// Collects link reference definitions and footnote labels, which may be used before they are defined.414    private func prescan(_ lines: [String]) {415        var fence: (Character, Int)?416        for raw in lines {417            var l = raw418            var guardCount = 0419            while let q = quoteStrip(l), guardCount < 64 { l = q; guardCount += 1 }420            if let f = fence {421                if let close = fenceStart(l), close.char == f.0, close.count >= f.1, close.info.isEmpty { fence = nil }422                continue423            }424            if let f = fenceStart(l) { fence = (f.char, f.count); continue }425            if let note = footnoteDefinition(l) { noteLabels.insert(note.label) }426            else if let ref = linkReference(l) {427                let key = MarkdownParser.normalizeLabel(ref.label)428                if references[key] == nil { references[key] = ref.url }429            }430        }431    }432433    // MARK: Blocks434435    private func inline(_ text: String) -> [Run] {436        var runs = MarkdownInlineParser(text, host: self).parse()437        // Trim white space at both ends of the paragraph (code spans keep theirs).438        func isText(_ r: Run) -> Bool { r.inline == nil && r.charStyle != StyleID.inlineCode }439        var lo = 0, hi = runs.count440        while lo < hi, isText(runs[lo]) {441            let t = String(runs[lo].text.drop { $0 == " " || $0 == "\t" })442            if t.isEmpty { lo += 1 } else { runs[lo].text = t; break }443        }444        while hi > lo, isText(runs[hi - 1]) {445            var t = runs[hi - 1].text446            while let c = t.last, c == " " || c == "\t" { t.removeLast() }447            if t.isEmpty { hi -= 1 } else { runs[hi - 1].text = t; break }448        }449        if lo > 0 || hi < runs.count { runs = Array(runs[lo..<hi]) }450        return runs451    }452453    private func paragraph(_ runs: [Run], style: String = StyleID.normal, _ ctx: Context) -> Paragraph {454        var p = Paragraph(style: style, runs: runs)455        if ctx.quote > 0, style == StyleID.normal {456            p.style = StyleID.quote457            if ctx.quote > 1 { p.props.indentLeft = 28 * Double(min(ctx.quote, 12)) }458        } else if ctx.inNote, style == StyleID.normal {459            p.style = StyleID.footnoteText460        }461        p.healRunBoundaries()462        return p463    }464465    private func codeParagraph(_ lines: [String], _ ctx: Context) -> Paragraph {466        let text = lines.joined(separator: lineSeparatorCharacter)467        return Paragraph(style: StyleID.code, runs: text.isEmpty ? [] : [Run(text)])468    }469470    func parseBlocks(_ lines: [String], _ ctx: Context) -> [Block] {471        var out: [Block] = []472        var i = 0473        while i < lines.count {474            let line = lines[i]475            if isBlank(line) { i += 1; continue }476477            if indent(line) >= 4 {478                var code: [String] = []479                while i < lines.count, isBlank(lines[i]) || indent(lines[i]) >= 4 { code.append(strip(lines[i], columns: 4)); i += 1 }480                while let last = code.last, isBlank(last) { code.removeLast() }481                out.append(.paragraph(codeParagraph(code, ctx)))482                continue483            }484            if let fence = fenceStart(line) {485                var code: [String] = []486                i += 1487                while i < lines.count {488                    if let close = fenceStart(lines[i]), close.char == fence.char, close.count >= fence.count, close.info.isEmpty, indent(lines[i]) < 4 { i += 1; break }489                    code.append(strip(lines[i], columns: fence.indent)); i += 1490                }491                if !fence.info.isEmpty { warnings.add(.dropped, "Code block language", "The language label of fenced code blocks is not kept.") }492                out.append(.paragraph(codeParagraph(code, ctx)))493                continue494            }495            if let math = mathBlock(lines, at: i) {496                out.append(.paragraph(paragraph([Run(inline: .equation(Equation(latex: math.latex, display: true)))], ctx)))497                i = math.next498                continue499            }500            if let heading = atxHeading(line) {501                out.append(.paragraph(headingParagraph(heading.text, level: heading.level, ctx)))502                i += 1503                continue504            }505            if simpleTableRule(line) == nil, let simple = simpleTable(lines, at: i, ctx) {506                out.append(.table(simple.table))507                i = simple.next508                continue509            }510            if isThematicBreak(line) {511                out.append(.paragraph(ModelIdiom.rule()))512                i += 1513                continue514            }515            if quoteStrip(line) != nil {516                var inner: [String] = []517                while i < lines.count {518                    if let stripped = quoteStrip(lines[i]) { inner.append(stripped); i += 1 }519                    else if !isBlank(lines[i]), let last = inner.last, !isBlank(last), !startsBlock(lines, at: i), listMarker(lines[i]) == nil { inner.append(lines[i]); i += 1 }520                    else { break }521                }522                if ctx.depth >= FormatLimits.maxMarkdownNesting {523                    warnings.add(.approximated, "Deep nesting", "Blocks nested deeper than \(FormatLimits.maxMarkdownNesting) levels are kept as plain text.")524                    out.append(.paragraph(paragraph([Run(inner.joined(separator: " "))], ctx)))525                    continue526                }527                var inside = ctx528                inside.depth += 1; inside.quote += 1529                let blocks = parseBlocks(inner, inside)530                if blocks.contains(where: { $0.paragraph?.style != StyleID.quote }) {531                    warnings.add(.approximated, "Block quotes", "Lists, code, headings and tables inside block quotes are kept without the quote formatting.")532                }533                if inside.quote > 1 { warnings.add(.approximated, "Block quotes", "Nested block quotes are written as indented quotes.") }534                out.append(contentsOf: blocks)535                continue536            }537            if let first = listMarker(line) {538                out.append(contentsOf: list(lines, &i, first, ctx))539                continue540            }541            if let note = footnoteDefinition(line) {542                var content = [note.rest]543                i += 1544                while i < lines.count {545                    let l = lines[i]546                    if isBlank(l) {547                        var j = i548                        while j < lines.count, isBlank(lines[j]) { j += 1 }549                        if j < lines.count, indent(lines[j]) >= 4 { content.append(contentsOf: Array(repeating: "", count: j - i)); i = j; continue }550                        break551                    }552                    if indent(l) >= 4 { content.append(strip(l, columns: 4)); i += 1; continue }553                    if let last = content.last, !isBlank(last), !startsBlock(lines, at: i), listMarker(l) == nil, footnoteDefinition(l) == nil, linkReference(l) == nil { content.append(trimLeading(l)); i += 1; continue }554                    break555                }556                var inside = Context()557                inside.depth = ctx.depth + 1; inside.inNote = true558                let blocks = ctx.depth >= FormatLimits.maxMarkdownNesting ? [Block.paragraph(Paragraph(content.joined(separator: " "), style: StyleID.footnoteText))] : parseBlocks(content, inside)559                if noteBodies[note.label] == nil { noteBodies[note.label] = flattenToParagraphs(blocks, feature: "Footnotes") }560                continue561            }562            if linkReference(line) != nil { i += 1; continue }563            let lead = trimLeading(line)564            if lead.hasPrefix(":::") {565                // pandoc fenced div. "::: {#refs}" marks where the bibliography goes; other divs are transparent.566                if lead.contains("#refs") {567                    var j = i + 1568                    while j < lines.count, !trimLeading(lines[j]).hasPrefix(":::") { j += 1 }569                    out.append(.bibliography(BibliographyBlock(title: nil)))570                    i = min(lines.count, j + 1)571                } else {572                    if lead.contains(where: { $0.isLetter }) { warnings.add(.dropped, "Fenced divs", "pandoc fenced div attributes are not kept; their content is.") }573                    i += 1574                }575                continue576            }577            if lead.hasPrefix("<!--") {578                var j = i579                var body = ""580                while j < lines.count {581                    body += lines[j]582                    if lines[j].contains("-->") { break }583                    j += 1584                }585                let inner = body.replacingOccurrences(of: "<!--", with: "").replacingOccurrences(of: "-->", with: "")586                if let close = lines[min(j, lines.count - 1)].range(of: "-->"), isBlank(String(lines[min(j, lines.count - 1)][close.upperBound...])) || j >= lines.count {587                    if !isBlank(inner) { warnings.add(.dropped, "HTML comments", "HTML comments in the Markdown source were dropped.") }588                    i = min(lines.count, j + 1)589                    continue590                }591            }592            if line.contains("|"), i + 1 < lines.count, let alignments = tableDelimiter(lines[i + 1]) {593                let header = splitRow(line)594                if header.count == alignments.count {595                    var rows = [header]596                    i += 2597                    while i < lines.count, !isBlank(lines[i]), !startsBlock(lines, at: i), rows.count * header.count < FormatLimits.maxTableCells { rows.append(splitRow(lines[i])); i += 1 }598                    out.append(.table(table(rows, alignments, ctx)))599                    continue600                }601            }602            // Paragraph (or setext heading).603            var text = [trimLeading(line)]604            i += 1605            var heading: Int?606            while i < lines.count {607                let l = lines[i]608                if isBlank(l) { break }609                if let level = setextLevel(l) { heading = level; i += 1; break }610                if startsBlock(lines, at: i) { break }611                if footnoteDefinition(l) != nil { break }612                text.append(trimLeading(l)); i += 1613            }614            if let level = heading {615                out.append(.paragraph(headingParagraph(text.map(trimmed).joined(separator: " "), level: level, ctx)))616            } else {617                out.append(.paragraph(paragraph(inline(text.joined(separator: "\n")), ctx)))618            }619        }620        return out621    }622623    private func headingParagraph(_ raw: String, level: Int, _ ctx: Context) -> Paragraph {624        var text = raw625        var anchor: String?626        // pandoc header attributes: "Title {#id .class}".627        if text.hasSuffix("}"), let open = text.lastIndex(of: "{") {628            let attributes = text[text.index(after: open)...].dropLast()629            if let id = attributes.split(separator: " ").first(where: { $0.hasPrefix("#") && $0.count > 1 }) { anchor = String(id.dropFirst()) }630            if anchor != nil || attributes.hasPrefix(".") || attributes == "-" { text = trimmed(String(text[..<open])) }631        }632        var runs = inline(text)633        if let anchor { runs.insert(Run(inline: .bookmark(TextSanitizer.clean(anchor))), at: 0) }634        var p = Paragraph(style: StyleID.heading(level), runs: runs)635        p.healRunBoundaries()636        return p637    }638639    private func list(_ lines: [String], _ i: inout Int, _ first: Marker, _ ctx: Context) -> [Block] {640        var out: [Block] = []641        var level = 0642        let listID: String643        if let parent = ctx.list {644            level = parent.level + 1645            if level > FormatLimits.maxListLevel {646                level = FormatLimits.maxListLevel647                warnings.add(.approximated, "Deep lists", "Lists nested deeper than \(FormatLimits.maxListLevel + 1) levels are flattened to the deepest level.")648            }649            listID = lists.sublist(parent: parent.id, level: level, ordered: first.ordered, start: first.number)650        } else {651            listID = lists.newList(ordered: first.ordered, start: first.number)652        }653        while i < lines.count, let marker = listMarker(lines[i]), sameList(marker, first), !(isThematicBreak(lines[i])) {654            var item = [marker.rest]655            i += 1656            while i < lines.count {657                let l = lines[i]658                if isBlank(l) {659                    var j = i660                    while j < lines.count, isBlank(lines[j]) { j += 1 }661                    if j < lines.count, indent(lines[j]) >= marker.contentOffset {662                        item.append(contentsOf: Array(repeating: "", count: j - i)); i = j663                        continue664                    }665                    break666                }667                if indent(l) >= marker.contentOffset { item.append(strip(l, columns: marker.contentOffset)); i += 1; continue }668                if let last = item.last, !isBlank(last), listMarker(l) == nil, !startsBlock(lines, at: i) { item.append(trimLeading(l)); i += 1; continue }669                break670            }671            var j = i672            while j < lines.count, isBlank(lines[j]) { j += 1 }673            if j < lines.count, let next = listMarker(lines[j]), sameList(next, first), !isThematicBreak(lines[j]) { i = j }674            out.append(contentsOf: listItem(item, listID: listID, level: level, ctx))675        }676        return out677    }678679    private func listItem(_ item: [String], listID: String, level: Int, _ ctx: Context) -> [Block] {680        var lines = item681        // GFM task list items: the checkbox becomes a ballot box character.682        for (mark, box) in [("[ ] ", "☐ "), ("[x] ", "☑ "), ("[X] ", "☑ ")] where lines[0].hasPrefix(mark) {683            lines[0] = box + lines[0].dropFirst(mark.count)684        }685        if lines[0] == "[ ]" { lines[0] = "☐" } else if lines[0] == "[x]" || lines[0] == "[X]" { lines[0] = "☑" }686        var blocks: [Block]687        if ctx.depth >= FormatLimits.maxMarkdownNesting {688            warnings.add(.approximated, "Deep nesting", "Blocks nested deeper than \(FormatLimits.maxMarkdownNesting) levels are kept as plain text.")689            blocks = [.paragraph(Paragraph(lines.map(trimmed).filter { !$0.isEmpty }.joined(separator: " ")))]690        } else {691            var inside = ctx692            inside.depth += 1693            inside.list = (listID, level)694            blocks = parseBlocks(lines, inside)695        }696        let indentLeft = 36 + 18 * Double(level)697        let plain: Set<String> = [StyleID.normal, StyleID.quote, StyleID.footnoteText]698        if case .paragraph(var p)? = blocks.first, p.list == nil, plain.contains(p.style), !ModelIdiom.isRule(p) {699            p.list = ListItem(listID: listID, level: level)700            p.style = StyleID.listParagraph701            p.props.indentLeft = nil702            blocks[0] = .paragraph(p)703        } else {704            blocks.insert(.paragraph(Paragraph(style: StyleID.listParagraph, list: ListItem(listID: listID, level: level))), at: 0)705        }706        for k in blocks.indices.dropFirst() {707            if case .paragraph(var p) = blocks[k], p.list == nil, plain.contains(p.style), p.props.indentLeft == nil, !ModelIdiom.isRule(p) {708                p.style = StyleID.listParagraph709                p.props.indentLeft = indentLeft710                blocks[k] = .paragraph(p)711            }712        }713        return blocks714    }715716    private func table(_ rows: [[String]], _ alignments: [TextAlignment?], _ ctx: Context) -> Table {717        let columns = alignments.count718        var out: [TableRow] = []719        for (r, row) in rows.enumerated() {720            var cells: [TableCell] = []721            for c in 0..<columns {722                var p = Paragraph(style: StyleID.tableText, runs: c < row.count ? inline(row[c]) : [])723                if let a = alignments[c] { p.props.alignment = a }724                p.healRunBoundaries()725                cells.append(TableCell(paragraphs: [p]))726            }727            if row.count > columns { warnings.add(.dropped, "Table cells", "Cells beyond the number of header columns were dropped.") }728            out.append(TableRow(cells: cells, isHeader: r == 0))729        }730        return Table(columnWidths: Array(repeating: 468 / Double(columns), count: columns), rows: out)731    }732733    private func flattenToParagraphs(_ blocks: [Block], feature: String) -> [Paragraph] {734        var out: [Paragraph] = []735        for block in blocks {736            switch block {737            case .paragraph(let p): out.append(p)738            case .table(let t):739                warnings.add(.approximated, feature, "Tables inside \(feature.lowercased()) are written as tab-separated paragraphs.")740                for row in t.rows {741                    var p = Paragraph(style: StyleID.footnoteText)742                    for (k, cell) in row.cells.enumerated() {743                        if k > 0 { p.runs.append(Run("\t")) }744                        for q in cell.paragraphs { p.runs.append(contentsOf: q.runs) }745                    }746                    out.append(p)747                }748            default: break749            }750        }751        return out.isEmpty ? [Paragraph(style: StyleID.footnoteText)] : out752    }753754    // MARK: Post-processing755756    /// Replaces temporary note references by real notes; references without a definition become literal text again.757    /// Only definitions reachable from the body are kept, so no reference can point at a note that is not in the document.758    private func resolveNotes(_ body: inout [Block], _ notes: inout [String: Note]) {759        let reachable = PendingNotes.reachable(from: body, definitions: noteBodies)760        let ids = Dictionary(uniqueKeysWithValues: reachable.map { ($0, DocID.make()) })761        func fix(_ p: inout Paragraph) {762            for k in p.runs.indices {763                guard case .note(let ref)? = p.runs[k].inline, let pending = PendingNotes.parse(ref) else { continue }764                if let id = ids[pending.target] { p.runs[k].inline = .note(NoteRef(kind: .footnote, id: id)) }765                else { p.runs[k] = Run("[^\(pending.target)]") }766            }767        }768        DocumentModel.mutateParagraphs(in: &body, fix)769        for (label, id) in ids {770            var paragraphs = noteBodies[label] ?? []771            for k in paragraphs.indices { fix(&paragraphs[k]) }772            notes[id] = Note(id: id, kind: .footnote, paragraphs: paragraphs)773        }774        let unused = noteBodies.count - reachable.count775        if unused > 0 { warnings.add(.dropped, "Footnotes", "\(unused) footnote definition\(unused == 1 ? " is" : "s are") never referenced and \(unused == 1 ? "was" : "were") dropped.") }776    }777778    /// Internal links ("#slug") that point at a heading get a bookmark on that heading, so they keep working.779    private func linkHeadingAnchors(_ body: inout [Block]) {780        var wanted: Set<String> = []781        var existing: Set<String> = []782        DocumentModel.forEachParagraph(in: body) { p, _ in783            for r in p.runs {784                if let link = r.props.link, link.hasPrefix("#"), link.count > 1 { wanted.insert(String(link.dropFirst())) }785                if case .bookmark(let name)? = r.inline { existing.insert(name) }786            }787        }788        wanted.subtract(existing)789        guard !wanted.isEmpty else { return }790        var slugger = Slugger()791        for k in body.indices {792            guard case .paragraph(var p) = body[k], p.style.hasPrefix("Heading") else { continue }793            let slug = slugger.unique(p.visibleText)794            if wanted.contains(slug) {795                p.runs.insert(Run(inline: .bookmark(slug)), at: 0)796                body[k] = .paragraph(p)797                wanted.remove(slug)798            }799        }800    }801}802
Files main
Menu

Theme