SPB Git

Repository Public

MarkdownParser.swift 15.9 KB 317 lines swift

1//2//  VQuant iOS3//  File: Core/Markdown/MarkdownParser.swift4//  Description: Block-level Markdown parser (GFM subset + LaTeX math) tolerant of text that is still being streamed.5//6//  Author: Simon-Pierre Boucher7//  Contact: contact@spboucher.ai8//910import Foundation1112enum ColumnAlignment: Hashable, Sendable { case leading, center, trailing }1314struct MarkdownListItem: Hashable, Sendable {15    /// `nil` for a plain item; `true` / `false` for a task-list item.16    var checked: Bool?17    var blocks: [MarkdownBlock]18}1920indirect enum MarkdownBlock: Hashable, Sendable {21    case heading(level: Int, text: String)22    case paragraph(String)23    /// `closed` is false while the closing fence has not arrived yet (streaming).24    case code(language: String?, code: String, closed: Bool)25    case quote([MarkdownBlock])26    case list(ordered: Bool, start: Int, items: [MarkdownListItem])27    case table(header: [String], alignments: [ColumnAlignment], rows: [[String]])28    case math(String)29    case rule30}3132/// Line-based parser for the Markdown that language models write: headings, paragraphs, fenced code, block quotes,33/// nested ordered / unordered / task lists, pipe tables, thematic breaks, and display math (`$$…$$`, `\[…\]`).34/// It never fails: unterminated constructs (a code fence or a table still being streamed) render as far as they go.35enum MarkdownParser {36    static func parse(_ text: String, depth: Int = 0) -> [MarkdownBlock] {37        let lines = text.replacingOccurrences(of: "\r\n", with: "\n").split(separator: "\n", omittingEmptySubsequences: false).map(String.init)38        var blocks: [MarkdownBlock] = []39        var i = 040        while i < lines.count {41            let line = lines[i]42            let trimmed = line.trimmingCharacters(in: .whitespaces)43            if trimmed.isEmpty { i += 1; continue }4445            if let fence = fenceOpening(line) {46                var body: [String] = []; var closed = false47                i += 148                while i < lines.count {49                    let t = lines[i].trimmingCharacters(in: .whitespaces)50                    if t.hasPrefix(fence.marker), t.allSatisfy({ $0 == fence.char }) { closed = true; i += 1; break }51                    body.append(dropIndent(lines[i], fence.indent)); i += 152                }53                blocks.append(.code(language: fence.language, code: body.joined(separator: "\n"), closed: closed))54                continue55            }5657            if let (math, next) = mathBlock(lines, at: i) { blocks.append(.math(math)); i = next; continue }5859            if let h = heading(trimmed) { blocks.append(.heading(level: h.level, text: h.text)); i += 1; continue }60            if isRule(trimmed) { blocks.append(.rule); i += 1; continue }6162            if trimmed.hasPrefix(">") {63                var inner: [String] = []64                while i < lines.count {65                    let t = lines[i].trimmingCharacters(in: .whitespaces)66                    guard t.hasPrefix(">") else { break }67                    var rest = t.dropFirst(); if rest.first == " " { rest = rest.dropFirst() }68                    inner.append(String(rest)); i += 169                }70                blocks.append(.quote(depth < 6 ? parse(inner.joined(separator: "\n"), depth: depth + 1) : [.paragraph(inner.joined(separator: "\n"))]))71                continue72            }7374            if trimmed.contains("|"), i + 1 < lines.count, let alignments = tableSeparator(lines[i + 1]) {75                let header = cells(trimmed)76                var rows: [[String]] = []77                i += 278                while i < lines.count {79                    let t = lines[i].trimmingCharacters(in: .whitespaces)80                    guard !t.isEmpty, t.contains("|") else { break }81                    var row = cells(t)82                    if row.count < header.count { row += Array(repeating: "", count: header.count - row.count) }83                    rows.append(Array(row.prefix(header.count))); i += 184                }85                var al = alignments86                if al.count < header.count { al += Array(repeating: .leading, count: header.count - al.count) }87                blocks.append(.table(header: header, alignments: Array(al.prefix(header.count)), rows: rows))88                continue89            }9091            if let first = listMarker(line) {92                let (list, next) = parseList(lines, at: i, first: first, depth: depth)93                blocks.append(list); i = next94                continue95            }9697            // paragraph: up to a blank line or the start of another block98            var para: [String] = [trimmed]99            i += 1100            while i < lines.count {101                let l = lines[i]; let t = l.trimmingCharacters(in: .whitespaces)102                if t.isEmpty || fenceOpening(l) != nil || heading(t) != nil || isRule(t) || t.hasPrefix(">") || t.hasPrefix("$$") || t.hasPrefix("\\[") || listMarker(l) != nil { break }103                if t.contains("|"), i + 1 < lines.count, tableSeparator(lines[i + 1]) != nil { break }104                para.append(t); i += 1105            }106            blocks.append(.paragraph(para.joined(separator: "\n")))107        }108        return blocks109    }110111    // MARK: fences112113    private static func fenceOpening(_ line: String) -> (marker: String, char: Character, language: String?, indent: Int)? {114        let indent = line.prefix { $0 == " " }.count115        guard indent <= 3 else { return nil }116        let t = line.dropFirst(indent)117        guard let c = t.first, c == "`" || c == "~" else { return nil }118        let run = t.prefix { $0 == c }.count119        guard run >= 3 else { return nil }120        let info = t.dropFirst(run).trimmingCharacters(in: .whitespaces)121        if c == "`", info.contains("`") { return nil }          // inline code such as ```x``` on one line122        let lang = info.split(separator: " ").first.map(String.init)123        return (String(repeating: c, count: run), c, lang?.isEmpty == false ? lang : nil, indent)124    }125126    private static func dropIndent(_ line: String, _ n: Int) -> String {127        var s = Substring(line); var k = n128        while k > 0, s.first == " " { s = s.dropFirst(); k -= 1 }129        return String(s)130    }131132    // MARK: math133134    private static func mathBlock(_ lines: [String], at start: Int) -> (String, Int)? {135        let t = lines[start].trimmingCharacters(in: .whitespaces)136        for (open, close) in [("$$", "$$"), ("\\[", "\\]")] where t.hasPrefix(open) {137            let afterOpen = String(t.dropFirst(open.count))138            if let r = afterOpen.range(of: close) {139                // single line: $$ … $$ (nothing meaningful may follow)140                guard afterOpen[r.upperBound...].trimmingCharacters(in: .whitespaces).isEmpty else { return nil }141                let body = String(afterOpen[..<r.lowerBound]).trimmingCharacters(in: .whitespaces)142                return body.isEmpty ? nil : (body, start + 1)143            }144            var body: [String] = afterOpen.trimmingCharacters(in: .whitespaces).isEmpty ? [] : [afterOpen]145            var i = start + 1146            while i < lines.count {147                let l = lines[i]148                if let r = l.range(of: close) {149                    let before = String(l[..<r.lowerBound])150                    if !before.trimmingCharacters(in: .whitespaces).isEmpty { body.append(before) }151                    return (body.joined(separator: "\n"), i + 1)152                }153                body.append(l); i += 1154            }155            // not closed yet (streaming): show what has arrived as math156            return body.isEmpty ? nil : (body.joined(separator: "\n"), i)157        }158        return nil159    }160161    // MARK: simple blocks162163    private static func heading(_ t: String) -> (level: Int, text: String)? {164        let level = t.prefix { $0 == "#" }.count165        guard (1...6).contains(level), t.dropFirst(level).first == " " else { return nil }166        var text = t.dropFirst(level).trimmingCharacters(in: .whitespaces)167        while text.hasSuffix("#") { text.removeLast() }168        return (level, text.trimmingCharacters(in: .whitespaces))169    }170171    private static func isRule(_ t: String) -> Bool {172        let s = t.filter { $0 != " " }173        guard s.count >= 3, let c = s.first, c == "-" || c == "*" || c == "_" else { return false }174        return s.allSatisfy { $0 == c }175    }176177    // MARK: tables178179    static func cells(_ line: String) -> [String] {180        var t = line.trimmingCharacters(in: .whitespaces)181        if t.hasPrefix("|") { t.removeFirst() }182        if t.hasSuffix("|"), !t.hasSuffix("\\|") { t.removeLast() }183        var out: [String] = []; var cur = ""; var escaped = false; var inCode = false184        for ch in t {185            if escaped { cur.append(ch == "|" ? "|" : "\\\(ch)"); escaped = false; continue }186            if ch == "\\" { escaped = true; continue }187            if ch == "`" { inCode.toggle() }188            if ch == "|" && !inCode { out.append(cur.trimmingCharacters(in: .whitespaces)); cur = "" } else { cur.append(ch) }189        }190        out.append(cur.trimmingCharacters(in: .whitespaces))191        return out192    }193194    private static func tableSeparator(_ line: String) -> [ColumnAlignment]? {195        let t = line.trimmingCharacters(in: .whitespaces)196        guard t.contains("-"), t.contains("|") || t.hasPrefix(":") || t.hasPrefix("-") else { return nil }197        guard t.contains("|") else { return nil }198        var out: [ColumnAlignment] = []199        for c in cells(t) {200            guard !c.isEmpty, c.allSatisfy({ $0 == "-" || $0 == ":" }), c.contains("-") else { return nil }201            out.append(c.hasPrefix(":") && c.hasSuffix(":") ? .center : c.hasSuffix(":") ? .trailing : .leading)202        }203        return out.isEmpty ? nil : out204    }205206    // MARK: lists207208    private struct Marker { var indent: Int; var ordered: Bool; var number: Int; var contentIndent: Int; var rest: String }209210    private static func listMarker(_ line: String) -> Marker? {211        var indent = 0212        var idx = line.startIndex213        while idx < line.endIndex, line[idx] == " " || line[idx] == "\t" { indent += line[idx] == "\t" ? 4 : 1; idx = line.index(after: idx) }214        let t = line[idx...]215        guard let c = t.first else { return nil }216        if c == "-" || c == "*" || c == "+" {217            let after = t.dropFirst()218            guard after.first == " " || after.first == "\t" else { return nil }219            let rest = after.drop { $0 == " " || $0 == "\t" }220            if c != "+", isRule(String(t)) { return nil }221            return Marker(indent: indent, ordered: false, number: 1, contentIndent: indent + 2, rest: String(rest))222        }223        let digits = t.prefix { $0.isASCII && $0.isNumber }224        guard !digits.isEmpty, digits.count <= 9 else { return nil }225        let after = t.dropFirst(digits.count)226        guard let p = after.first, p == "." || p == ")", after.dropFirst().first == " " else { return nil }227        let rest = after.dropFirst().drop { $0 == " " }228        return Marker(indent: indent, ordered: true, number: Int(digits) ?? 1, contentIndent: indent + digits.count + 2, rest: String(rest))229    }230231    private static func parseList(_ lines: [String], at start: Int, first: Marker, depth: Int) -> (MarkdownBlock, Int) {232        var items: [MarkdownListItem] = []233        var i = start234        let base = first.indent235        while i < lines.count, let m = listMarker(lines[i]), m.indent <= base + 1, m.ordered == first.ordered {236            var content: [String] = [m.rest]237            i += 1238            while i < lines.count {239                let l = lines[i]240                let blank = l.trimmingCharacters(in: .whitespaces).isEmpty241                let indent = l.prefix { $0 == " " || $0 == "\t" }.reduce(0) { $0 + ($1 == "\t" ? 4 : 1) }242                if blank {243                    // a blank line stays in the item only when indented content follows244                    var j = i + 1245                    while j < lines.count, lines[j].trimmingCharacters(in: .whitespaces).isEmpty { j += 1 }246                    guard j < lines.count else { break }247                    let nextIndent = lines[j].prefix { $0 == " " || $0 == "\t" }.reduce(0) { $0 + ($1 == "\t" ? 4 : 1) }248                    if nextIndent >= base + 2 { content.append(""); i += 1; continue }249                    break250                }251                if let nm = listMarker(l), nm.indent <= base + 1 { break }               // next sibling (or another list)252                if indent >= base + 2 { content.append(dropIndentMixed(l, m.contentIndent)); i += 1; continue }253                // lazy continuation of the item's paragraph, unless another block starts here254                let t = l.trimmingCharacters(in: .whitespaces)255                if fenceOpening(l) != nil || heading(t) != nil || isRule(t) || t.hasPrefix(">") || t.hasPrefix("$$") { break }256                content.append(t); i += 1257            }258            var body = content.joined(separator: "\n")259            var checked: Bool?260            for (prefix, value) in [("[ ] ", false), ("[x] ", true), ("[X] ", true)] where body.hasPrefix(prefix) { checked = value; body = String(body.dropFirst(prefix.count)); break }261            items.append(MarkdownListItem(checked: checked, blocks: depth < 6 ? parse(body, depth: depth + 1) : [.paragraph(body)]))262            // blank lines between sibling items263            var j = i264            while j < lines.count, lines[j].trimmingCharacters(in: .whitespaces).isEmpty { j += 1 }265            if j < lines.count, let nm = listMarker(lines[j]), nm.indent <= base + 1, nm.ordered == first.ordered { i = j } else { break }266        }267        return (.list(ordered: first.ordered, start: first.number, items: items), i)268    }269270    private static func dropIndentMixed(_ line: String, _ n: Int) -> String {271        var s = Substring(line); var k = n272        while k > 0, let c = s.first, c == " " || c == "\t" { s = s.dropFirst(); k -= c == "\t" ? 4 : 1 }273        return String(s)274    }275}276277/// Splits a line of text into plain Markdown and inline math (`$…$`, `\(…\)`).278enum InlineMath {279    enum Segment: Hashable, Sendable { case text(String), math(String) }280281    /// `$` opens math only when followed by a non-space, and closes only when preceded by a non-space and not followed282    /// by a digit (the pandoc rule): "5 $ et 10 $" and "$5 and $10" stay text.283    static func split(_ s: String) -> [Segment] {284        guard s.contains("$") || s.contains("\\(") else { return [.text(s)] }285        var out: [Segment] = []; var text = ""286        let chars = Array(s); var i = 0; var inCode = false287        func flush() { if !text.isEmpty { out.append(.text(text)); text = "" } }288        while i < chars.count {289            let c = chars[i]290            if c == "`" { inCode.toggle(); text.append(c); i += 1; continue }291            if inCode { text.append(c); i += 1; continue }292            if c == "\\", i + 1 < chars.count, chars[i + 1] == "(" {293                if let end = find(chars, from: i + 2, close: ["\\", ")"]) { flush(); out.append(.math(String(chars[(i + 2)..<end]).trimmingCharacters(in: .whitespaces))); i = end + 2; continue }294            }295            if c == "\\", i + 1 < chars.count, chars[i + 1] == "$" { text.append("$"); i += 2; continue }296            if c == "$", i + 1 < chars.count, chars[i + 1] != " ", chars[i + 1] != "$", chars[i + 1] != "\n" {297                var j = i + 1; var found: Int?298                while j < chars.count, chars[j] != "\n" {299                    if chars[j] == "\\" { j += 2; continue }300                    if chars[j] == "$", chars[j - 1] != " ", !(j + 1 < chars.count && chars[j + 1].isNumber) { found = j; break }301                    j += 1302                }303                if let found, found > i + 1 { flush(); out.append(.math(String(chars[(i + 1)..<found]))); i = found + 1; continue }304            }305            text.append(c); i += 1306        }307        flush()308        return out309    }310311    private static func find(_ chars: [Character], from: Int, close: [Character]) -> Int? {312        var j = from313        while j + 1 < chars.count { if chars[j] == close[0], chars[j + 1] == close[1] { return j }; j += 1 }314        return nil315    }316}317
Files main
Menu

Theme