Repository Public
1//2// VQuant iOS3// File: Core/Markdown/MarkdownParser.swift4// Description: Block-level Markdown parser (GFM subset + LaTeX math) tolerant of text that is still being streamed.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8//910import Foundation1112enum ColumnAlignment: Hashable, Sendable { case leading, center, trailing }1314struct MarkdownListItem: Hashable, Sendable {15 /// `nil` for a plain item; `true` / `false` for a task-list item.16 var checked: Bool?17 var blocks: [MarkdownBlock]18}1920indirect enum MarkdownBlock: Hashable, Sendable {21 case heading(level: Int, text: String)22 case paragraph(String)23 /// `closed` is false while the closing fence has not arrived yet (streaming).24 case code(language: String?, code: String, closed: Bool)25 case quote([MarkdownBlock])26 case list(ordered: Bool, start: Int, items: [MarkdownListItem])27 case table(header: [String], alignments: [ColumnAlignment], rows: [[String]])28 case math(String)29 case rule30}3132/// Line-based parser for the Markdown that language models write: headings, paragraphs, fenced code, block quotes,33/// nested ordered / unordered / task lists, pipe tables, thematic breaks, and display math (`$$…$$`, `\[…\]`).34/// It never fails: unterminated constructs (a code fence or a table still being streamed) render as far as they go.35enum MarkdownParser {36 static func parse(_ text: String, depth: Int = 0) -> [MarkdownBlock] {37 let lines = text.replacingOccurrences(of: "\r\n", with: "\n").split(separator: "\n", omittingEmptySubsequences: false).map(String.init)38 var blocks: [MarkdownBlock] = []39 var i = 040 while i < lines.count {41 let line = lines[i]42 let trimmed = line.trimmingCharacters(in: .whitespaces)43 if trimmed.isEmpty { i += 1; continue }4445 if let fence = fenceOpening(line) {46 var body: [String] = []; var closed = false47 i += 148 while i < lines.count {49 let t = lines[i].trimmingCharacters(in: .whitespaces)50 if t.hasPrefix(fence.marker), t.allSatisfy({ $0 == fence.char }) { closed = true; i += 1; break }51 body.append(dropIndent(lines[i], fence.indent)); i += 152 }53 blocks.append(.code(language: fence.language, code: body.joined(separator: "\n"), closed: closed))54 continue55 }5657 if let (math, next) = mathBlock(lines, at: i) { blocks.append(.math(math)); i = next; continue }5859 if let h = heading(trimmed) { blocks.append(.heading(level: h.level, text: h.text)); i += 1; continue }60 if isRule(trimmed) { blocks.append(.rule); i += 1; continue }6162 if trimmed.hasPrefix(">") {63 var inner: [String] = []64 while i < lines.count {65 let t = lines[i].trimmingCharacters(in: .whitespaces)66 guard t.hasPrefix(">") else { break }67 var rest = t.dropFirst(); if rest.first == " " { rest = rest.dropFirst() }68 inner.append(String(rest)); i += 169 }70 blocks.append(.quote(depth < 6 ? parse(inner.joined(separator: "\n"), depth: depth + 1) : [.paragraph(inner.joined(separator: "\n"))]))71 continue72 }7374 if trimmed.contains("|"), i + 1 < lines.count, let alignments = tableSeparator(lines[i + 1]) {75 let header = cells(trimmed)76 var rows: [[String]] = []77 i += 278 while i < lines.count {79 let t = lines[i].trimmingCharacters(in: .whitespaces)80 guard !t.isEmpty, t.contains("|") else { break }81 var row = cells(t)82 if row.count < header.count { row += Array(repeating: "", count: header.count - row.count) }83 rows.append(Array(row.prefix(header.count))); i += 184 }85 var al = alignments86 if al.count < header.count { al += Array(repeating: .leading, count: header.count - al.count) }87 blocks.append(.table(header: header, alignments: Array(al.prefix(header.count)), rows: rows))88 continue89 }9091 if let first = listMarker(line) {92 let (list, next) = parseList(lines, at: i, first: first, depth: depth)93 blocks.append(list); i = next94 continue95 }9697 // paragraph: up to a blank line or the start of another block98 var para: [String] = [trimmed]99 i += 1100 while i < lines.count {101 let l = lines[i]; let t = l.trimmingCharacters(in: .whitespaces)102 if t.isEmpty || fenceOpening(l) != nil || heading(t) != nil || isRule(t) || t.hasPrefix(">") || t.hasPrefix("$$") || t.hasPrefix("\\[") || listMarker(l) != nil { break }103 if t.contains("|"), i + 1 < lines.count, tableSeparator(lines[i + 1]) != nil { break }104 para.append(t); i += 1105 }106 blocks.append(.paragraph(para.joined(separator: "\n")))107 }108 return blocks109 }110111 // MARK: fences112113 private static func fenceOpening(_ line: String) -> (marker: String, char: Character, language: String?, indent: Int)? {114 let indent = line.prefix { $0 == " " }.count115 guard indent <= 3 else { return nil }116 let t = line.dropFirst(indent)117 guard let c = t.first, c == "`" || c == "~" else { return nil }118 let run = t.prefix { $0 == c }.count119 guard run >= 3 else { return nil }120 let info = t.dropFirst(run).trimmingCharacters(in: .whitespaces)121 if c == "`", info.contains("`") { return nil } // inline code such as ```x``` on one line122 let lang = info.split(separator: " ").first.map(String.init)123 return (String(repeating: c, count: run), c, lang?.isEmpty == false ? lang : nil, indent)124 }125126 private static func dropIndent(_ line: String, _ n: Int) -> String {127 var s = Substring(line); var k = n128 while k > 0, s.first == " " { s = s.dropFirst(); k -= 1 }129 return String(s)130 }131132 // MARK: math133134 private static func mathBlock(_ lines: [String], at start: Int) -> (String, Int)? {135 let t = lines[start].trimmingCharacters(in: .whitespaces)136 for (open, close) in [("$$", "$$"), ("\\[", "\\]")] where t.hasPrefix(open) {137 let afterOpen = String(t.dropFirst(open.count))138 if let r = afterOpen.range(of: close) {139 // single line: $$ … $$ (nothing meaningful may follow)140 guard afterOpen[r.upperBound...].trimmingCharacters(in: .whitespaces).isEmpty else { return nil }141 let body = String(afterOpen[..<r.lowerBound]).trimmingCharacters(in: .whitespaces)142 return body.isEmpty ? nil : (body, start + 1)143 }144 var body: [String] = afterOpen.trimmingCharacters(in: .whitespaces).isEmpty ? [] : [afterOpen]145 var i = start + 1146 while i < lines.count {147 let l = lines[i]148 if let r = l.range(of: close) {149 let before = String(l[..<r.lowerBound])150 if !before.trimmingCharacters(in: .whitespaces).isEmpty { body.append(before) }151 return (body.joined(separator: "\n"), i + 1)152 }153 body.append(l); i += 1154 }155 // not closed yet (streaming): show what has arrived as math156 return body.isEmpty ? nil : (body.joined(separator: "\n"), i)157 }158 return nil159 }160161 // MARK: simple blocks162163 private static func heading(_ t: String) -> (level: Int, text: String)? {164 let level = t.prefix { $0 == "#" }.count165 guard (1...6).contains(level), t.dropFirst(level).first == " " else { return nil }166 var text = t.dropFirst(level).trimmingCharacters(in: .whitespaces)167 while text.hasSuffix("#") { text.removeLast() }168 return (level, text.trimmingCharacters(in: .whitespaces))169 }170171 private static func isRule(_ t: String) -> Bool {172 let s = t.filter { $0 != " " }173 guard s.count >= 3, let c = s.first, c == "-" || c == "*" || c == "_" else { return false }174 return s.allSatisfy { $0 == c }175 }176177 // MARK: tables178179 static func cells(_ line: String) -> [String] {180 var t = line.trimmingCharacters(in: .whitespaces)181 if t.hasPrefix("|") { t.removeFirst() }182 if t.hasSuffix("|"), !t.hasSuffix("\\|") { t.removeLast() }183 var out: [String] = []; var cur = ""; var escaped = false; var inCode = false184 for ch in t {185 if escaped { cur.append(ch == "|" ? "|" : "\\\(ch)"); escaped = false; continue }186 if ch == "\\" { escaped = true; continue }187 if ch == "`" { inCode.toggle() }188 if ch == "|" && !inCode { out.append(cur.trimmingCharacters(in: .whitespaces)); cur = "" } else { cur.append(ch) }189 }190 out.append(cur.trimmingCharacters(in: .whitespaces))191 return out192 }193194 private static func tableSeparator(_ line: String) -> [ColumnAlignment]? {195 let t = line.trimmingCharacters(in: .whitespaces)196 guard t.contains("-"), t.contains("|") || t.hasPrefix(":") || t.hasPrefix("-") else { return nil }197 guard t.contains("|") else { return nil }198 var out: [ColumnAlignment] = []199 for c in cells(t) {200 guard !c.isEmpty, c.allSatisfy({ $0 == "-" || $0 == ":" }), c.contains("-") else { return nil }201 out.append(c.hasPrefix(":") && c.hasSuffix(":") ? .center : c.hasSuffix(":") ? .trailing : .leading)202 }203 return out.isEmpty ? nil : out204 }205206 // MARK: lists207208 private struct Marker { var indent: Int; var ordered: Bool; var number: Int; var contentIndent: Int; var rest: String }209210 private static func listMarker(_ line: String) -> Marker? {211 var indent = 0212 var idx = line.startIndex213 while idx < line.endIndex, line[idx] == " " || line[idx] == "\t" { indent += line[idx] == "\t" ? 4 : 1; idx = line.index(after: idx) }214 let t = line[idx...]215 guard let c = t.first else { return nil }216 if c == "-" || c == "*" || c == "+" {217 let after = t.dropFirst()218 guard after.first == " " || after.first == "\t" else { return nil }219 let rest = after.drop { $0 == " " || $0 == "\t" }220 if c != "+", isRule(String(t)) { return nil }221 return Marker(indent: indent, ordered: false, number: 1, contentIndent: indent + 2, rest: String(rest))222 }223 let digits = t.prefix { $0.isASCII && $0.isNumber }224 guard !digits.isEmpty, digits.count <= 9 else { return nil }225 let after = t.dropFirst(digits.count)226 guard let p = after.first, p == "." || p == ")", after.dropFirst().first == " " else { return nil }227 let rest = after.dropFirst().drop { $0 == " " }228 return Marker(indent: indent, ordered: true, number: Int(digits) ?? 1, contentIndent: indent + digits.count + 2, rest: String(rest))229 }230231 private static func parseList(_ lines: [String], at start: Int, first: Marker, depth: Int) -> (MarkdownBlock, Int) {232 var items: [MarkdownListItem] = []233 var i = start234 let base = first.indent235 while i < lines.count, let m = listMarker(lines[i]), m.indent <= base + 1, m.ordered == first.ordered {236 var content: [String] = [m.rest]237 i += 1238 while i < lines.count {239 let l = lines[i]240 let blank = l.trimmingCharacters(in: .whitespaces).isEmpty241 let indent = l.prefix { $0 == " " || $0 == "\t" }.reduce(0) { $0 + ($1 == "\t" ? 4 : 1) }242 if blank {243 // a blank line stays in the item only when indented content follows244 var j = i + 1245 while j < lines.count, lines[j].trimmingCharacters(in: .whitespaces).isEmpty { j += 1 }246 guard j < lines.count else { break }247 let nextIndent = lines[j].prefix { $0 == " " || $0 == "\t" }.reduce(0) { $0 + ($1 == "\t" ? 4 : 1) }248 if nextIndent >= base + 2 { content.append(""); i += 1; continue }249 break250 }251 if let nm = listMarker(l), nm.indent <= base + 1 { break } // next sibling (or another list)252 if indent >= base + 2 { content.append(dropIndentMixed(l, m.contentIndent)); i += 1; continue }253 // lazy continuation of the item's paragraph, unless another block starts here254 let t = l.trimmingCharacters(in: .whitespaces)255 if fenceOpening(l) != nil || heading(t) != nil || isRule(t) || t.hasPrefix(">") || t.hasPrefix("$$") { break }256 content.append(t); i += 1257 }258 var body = content.joined(separator: "\n")259 var checked: Bool?260 for (prefix, value) in [("[ ] ", false), ("[x] ", true), ("[X] ", true)] where body.hasPrefix(prefix) { checked = value; body = String(body.dropFirst(prefix.count)); break }261 items.append(MarkdownListItem(checked: checked, blocks: depth < 6 ? parse(body, depth: depth + 1) : [.paragraph(body)]))262 // blank lines between sibling items263 var j = i264 while j < lines.count, lines[j].trimmingCharacters(in: .whitespaces).isEmpty { j += 1 }265 if j < lines.count, let nm = listMarker(lines[j]), nm.indent <= base + 1, nm.ordered == first.ordered { i = j } else { break }266 }267 return (.list(ordered: first.ordered, start: first.number, items: items), i)268 }269270 private static func dropIndentMixed(_ line: String, _ n: Int) -> String {271 var s = Substring(line); var k = n272 while k > 0, let c = s.first, c == " " || c == "\t" { s = s.dropFirst(); k -= c == "\t" ? 4 : 1 }273 return String(s)274 }275}276277/// Splits a line of text into plain Markdown and inline math (`$…$`, `\(…\)`).278enum InlineMath {279 enum Segment: Hashable, Sendable { case text(String), math(String) }280281 /// `$` opens math only when followed by a non-space, and closes only when preceded by a non-space and not followed282 /// by a digit (the pandoc rule): "5 $ et 10 $" and "$5 and $10" stay text.283 static func split(_ s: String) -> [Segment] {284 guard s.contains("$") || s.contains("\\(") else { return [.text(s)] }285 var out: [Segment] = []; var text = ""286 let chars = Array(s); var i = 0; var inCode = false287 func flush() { if !text.isEmpty { out.append(.text(text)); text = "" } }288 while i < chars.count {289 let c = chars[i]290 if c == "`" { inCode.toggle(); text.append(c); i += 1; continue }291 if inCode { text.append(c); i += 1; continue }292 if c == "\\", i + 1 < chars.count, chars[i + 1] == "(" {293 if let end = find(chars, from: i + 2, close: ["\\", ")"]) { flush(); out.append(.math(String(chars[(i + 2)..<end]).trimmingCharacters(in: .whitespaces))); i = end + 2; continue }294 }295 if c == "\\", i + 1 < chars.count, chars[i + 1] == "$" { text.append("$"); i += 2; continue }296 if c == "$", i + 1 < chars.count, chars[i + 1] != " ", chars[i + 1] != "$", chars[i + 1] != "\n" {297 var j = i + 1; var found: Int?298 while j < chars.count, chars[j] != "\n" {299 if chars[j] == "\\" { j += 2; continue }300 if chars[j] == "$", chars[j - 1] != " ", !(j + 1 < chars.count && chars[j + 1].isNumber) { found = j; break }301 j += 1302 }303 if let found, found > i + 1 { flush(); out.append(.math(String(chars[(i + 1)..<found]))); i = found + 1; continue }304 }305 text.append(c); i += 1306 }307 flush()308 return out309 }310311 private static func find(_ chars: [Character], from: Int, close: [Character]) -> Int? {312 var j = from313 while j + 1 < chars.count { if chars[j] == close[0], chars[j + 1] == close[1] { return j }; j += 1 }314 return nil315 }316}317