Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatUI/Sources/Markdown/MarkdownModel.swift3// Description: Render model for assistant text (blocks, inlines, math) and the Markdown → model parser with incremental chunking for streaming.4import Foundation5import Markdown67// MARK: - Render model89public indirect enum MDInline: Sendable, Hashable {10 case text(String)11 case emphasis([MDInline])12 case strong([MDInline])13 case strikethrough([MDInline])14 case code(String)15 case link(destination: String, children: [MDInline])16 case image(source: String, alt: String)17 case math(String, display: Bool)18 case lineBreak19 case softBreak2021 /// Text as a reader would copy it (math keeps its TeX source).22 public var plainText: String {23 switch self {24 case .text(let s), .code(let s): s25 case .emphasis(let c), .strong(let c), .strikethrough(let c): c.map(\.plainText).joined()26 case .link(_, let c): c.map(\.plainText).joined()27 case .image(_, let alt): alt28 case .math(let tex, let display): display ? "$$\(tex)$$" : "$\(tex)$"29 case .lineBreak: "\n"30 case .softBreak: " "31 }32 }33}3435public struct MDListItem: Sendable, Hashable {36 public var checkbox: Bool?37 public var blocks: [MDBlock]38}3940public struct MDTable: Sendable, Hashable {41 public enum Alignment: Sendable, Hashable { case leading, center, trailing }42 public var alignments: [Alignment]43 public var header: [[MDInline]]44 public var rows: [[[MDInline]]]45 public var columnCount: Int { max(header.count, rows.map(\.count).max() ?? 0) }46}4748public indirect enum MDBlock: Sendable, Hashable {49 case paragraph([MDInline])50 case heading(level: Int, [MDInline])51 case code(language: String?, text: String)52 case quote([MDBlock])53 case list(ordered: Bool, start: Int, items: [MDListItem])54 case table(MDTable)55 case rule56 case math(String)57 /// Raw HTML from the model is never interpreted: it is displayed as literal text.58 case html(String)5960 public var plainText: String {61 switch self {62 case .paragraph(let i), .heading(_, let i): i.map(\.plainText).joined()63 case .code(_, let t): t64 case .quote(let b): b.map(\.plainText).joined(separator: "\n")65 case .list(let ordered, let start, let items):66 items.enumerated().map { n, it in (ordered ? "\(start + n). " : "• ") + it.blocks.map(\.plainText).joined(separator: "\n") }.joined(separator: "\n")67 case .table(let t): ([t.header] + t.rows).map { $0.map { $0.map(\.plainText).joined() }.joined(separator: "\t") }.joined(separator: "\n")68 case .rule: "———"69 case .math(let tex): "$$\(tex)$$"70 case .html(let s): s71 }72 }73}7475/// One independently parsed slice of a message. While a message streams only its last chunk changes, so the views of76/// every earlier chunk are left untouched.77public struct MDChunk: Sendable, Hashable, Identifiable {78 public let id: Int79 public let source: String80 public let blocks: [MDBlock]81}8283// MARK: - Parser8485public final class MarkdownParser: @unchecked Sendable {86 public static let shared = MarkdownParser()87 private let lock = NSLock()88 private var cache: [String: [MDBlock]] = [:]89 private var order: [String] = []90 private let capacity: Int9192 public init(capacity: Int = 6000) { self.capacity = capacity }9394 /// Splits `source` into top-level chunks and parses each (cached by content).95 public func chunks(_ source: String) -> [MDChunk] {96 Self.split(source).enumerated().map { i, s in MDChunk(id: i, source: s, blocks: blocks(forChunk: s)) }97 }9899 /// Parses a whole document in one piece (exports, tests).100 public func parse(_ source: String) -> [MDBlock] { Self.parseUncached(source) }101102 private func blocks(forChunk s: String) -> [MDBlock] {103 if let hit = lock.withLock({ cache[s] }) { return hit }104 let parsed = Self.parseUncached(s)105 lock.withLock {106 if cache[s] == nil { order.append(s) }107 cache[s] = parsed108 if order.count > capacity { for k in order.prefix(capacity / 4) { cache[k] = nil }; order.removeFirst(capacity / 4) }109 }110 return parsed111 }112113 /// Cuts at blank lines that are outside fenced code and display math, and never between the lines of a list or a114 /// table (those keep their own blank-line semantics inside one chunk when they are contiguous).115 static func split(_ source: String) -> [String] {116 var chunks: [String] = []; var current: [Substring] = []117 var fence: (char: Character, count: Int)?; var inDisplayMath = false118 func flush() { let s = current.joined(separator: "\n"); if !s.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { chunks.append(s) }; current.removeAll(keepingCapacity: true) }119 for line in source.split(separator: "\n", omittingEmptySubsequences: false) {120 let trimmed = line.drop(while: { $0 == " " })121 if let f = fence {122 current.append(line)123 if trimmed.hasPrefix(String(repeating: f.char, count: f.count)), trimmed.allSatisfy({ $0 == f.char || $0 == " " }) { fence = nil }124 continue125 }126 if let c = trimmed.first, c == "`" || c == "~" {127 let n = trimmed.prefix(while: { $0 == c }).count128 if n >= 3, line.count - trimmed.count < 4 {129 // a fence that follows a list item stays with it; otherwise it starts its own chunk130 if let prev = current.last, prev.trimmingCharacters(in: .whitespaces).isEmpty { flush() }131 fence = (c, n); current.append(line); continue132 }133 }134 let dollars = line.components(separatedBy: "$$").count - 1135 if dollars % 2 == 1 { inDisplayMath.toggle() }136 if trimmed.hasPrefix("\\[") && !line.contains("\\]") { inDisplayMath = true }137 if inDisplayMath && line.contains("\\]") && !trimmed.hasPrefix("\\[") { inDisplayMath = false }138 if line.allSatisfy({ $0 == " " || $0 == "\t" }) && !inDisplayMath { flush() } else { current.append(line) }139 }140 flush()141 return chunks142 }143144 // MARK: Math protection145146 private static let open: Character = "\u{E000}", close: Character = "\u{E001}"147148 struct Protected { var text: String; var math: [(tex: String, display: Bool, raw: String)] }149150 /// Replaces TeX spans by private-use placeholders so that the Markdown parser cannot mangle `_`, `*` or `\\` in them.151 /// Code spans and fenced blocks are skipped. `$…$` follows the usual rule: no space just inside the delimiters and152 /// no digit right after the closing one (so "$5 and $10" stays text).153 static func protectMath(_ source: String) -> Protected {154 var out = ""; out.reserveCapacity(source.count)155 var math: [(String, Bool, String)] = []156 let s = Array(source); var i = 0; let n = s.count157 var atLineStart = true158 func emit(_ tex: String, _ display: Bool, _ raw: String) { out.append(open); out.append(String(math.count)); out.append(close); math.append((tex.trimmingCharacters(in: .whitespacesAndNewlines), display, raw)) }159 func find(_ needle: [Character], from: Int, sameLine: Bool) -> Int? {160 var j = from161 while j + needle.count <= n {162 if sameLine && s[j] == "\n" { return nil }163 if s[j] == "\\" && needle != ["\\", ")"] && needle != ["\\", "]"] { j += 2; continue }164 if Array(s[j..<(j + needle.count)]) == needle { return j }165 j += 1166 }167 return nil168 }169 while i < n {170 let c = s[i]171 if atLineStart {172 // fenced code block: copy verbatim up to the closing fence173 var k = i; while k < n, s[k] == " " { k += 1 }174 if k < n, s[k] == "`" || s[k] == "~", k - i < 4 {175 let fc = s[k]; var run = 0; while k + run < n, s[k + run] == fc { run += 1 }176 if run >= 3 {177 var j = k + run; var closed = false178 while j < n {179 guard let nl = s[j...].firstIndex(of: "\n") else { j = n; break }180 var p = nl + 1; while p < n, s[p] == " " { p += 1 }181 var r = 0; while p + r < n, s[p + r] == fc { r += 1 }182 if r >= run { var e = p + r; while e < n, s[e] != "\n" { e += 1 }; j = e; closed = true; break }183 j = nl + 1184 }185 _ = closed186 out.append(contentsOf: s[i..<j]); i = j; atLineStart = false; continue187 }188 }189 }190 atLineStart = c == "\n"191 if c == "`" { // inline code span: same-length backtick run closes it192 var run = 0; while i + run < n, s[i + run] == "`" { run += 1 }193 var j = i + run; var end: Int?194 while j < n { if s[j] == "`" { var r = 0; while j + r < n, s[j + r] == "`" { r += 1 }; if r == run { end = j + r; break }; j += r } else { j += 1 } }195 let stop = end ?? (i + run)196 out.append(contentsOf: s[i..<stop]); i = stop; continue197 }198 if c == "\\", i + 1 < n {199 if s[i + 1] == "[", let e = find(["\\", "]"], from: i + 2, sameLine: false) { emit(String(s[(i + 2)..<e]), true, String(s[i..<(e + 2)])); i = e + 2; continue }200 if s[i + 1] == "(", let e = find(["\\", ")"], from: i + 2, sameLine: false) { emit(String(s[(i + 2)..<e]), false, String(s[i..<(e + 2)])); i = e + 2; continue }201 out.append(c); out.append(s[i + 1]); i += 2; continue202 }203 if c == "$" {204 if i + 1 < n, s[i + 1] == "$" {205 if let e = find(["$", "$"], from: i + 2, sameLine: false) { emit(String(s[(i + 2)..<e]), true, String(s[i..<(e + 2)])); i = e + 2; continue }206 } else if i + 1 < n, !s[i + 1].isWhitespace, let e = find(["$"], from: i + 1, sameLine: true), e > i + 1, !s[e - 1].isWhitespace, !(e + 1 < n && s[e + 1].isNumber), !(i > 0 && s[i - 1].isNumber),207 !s[i + 1].isNumber || s[(i + 1)..<e].contains(where: { "\\^_{}=+*/<>".contains($0) }) || !s[(i + 1)..<e].contains(" ") {208 emit(String(s[(i + 1)..<e]), false, String(s[i...e])); i = e + 1; continue209 }210 }211 out.append(c); i += 1212 }213 return Protected(text: out, math: math)214 }215216 // MARK: Conversion217218 static func parseUncached(_ source: String) -> [MDBlock] {219 let protected = protectMath(source)220 let doc = Document(parsing: protected.text, options: [.disableSmartOpts])221 var conv = Converter(math: protected.math)222 return doc.children.flatMap { conv.block($0) }223 }224225 private struct Converter {226 let math: [(tex: String, display: Bool, raw: String)]227228 /// Puts the original TeX source back (inside code, where placeholders must not survive).229 func restore(_ s: String) -> String {230 guard s.contains(MarkdownParser.open) else { return s }231 var out = ""; var rest = Substring(s)232 while let o = rest.firstIndex(of: MarkdownParser.open), let c = rest[o...].firstIndex(of: MarkdownParser.close) {233 out += rest[..<o]234 if let idx = Int(rest[rest.index(after: o)..<c]), math.indices.contains(idx) { out += math[idx].raw }235 rest = rest[rest.index(after: c)...]236 }237 return out + rest238 }239240 func splitText(_ s: String) -> [MDInline] {241 guard s.contains(MarkdownParser.open) else { return s.isEmpty ? [] : [.text(s)] }242 var out: [MDInline] = []; var rest = Substring(s)243 while let o = rest.firstIndex(of: MarkdownParser.open), let c = rest[o...].firstIndex(of: MarkdownParser.close) {244 if o > rest.startIndex { out.append(.text(String(rest[..<o]))) }245 if let idx = Int(rest[rest.index(after: o)..<c]), math.indices.contains(idx) { out.append(.math(math[idx].tex, display: math[idx].display)) }246 rest = rest[rest.index(after: c)...]247 }248 if !rest.isEmpty { out.append(.text(String(rest))) }249 return out250 }251252 func inlines(_ node: Markup) -> [MDInline] { node.children.flatMap { inline($0) } }253254 func inline(_ node: Markup) -> [MDInline] {255 switch node {256 case let t as Markdown.Text: return splitText(t.string)257 case let e as Emphasis: return [.emphasis(inlines(e))]258 case let s as Strong: return [.strong(inlines(s))]259 case let s as Strikethrough: return [.strikethrough(inlines(s))]260 case let c as InlineCode: return [.code(restore(c.code))]261 case let l as Markdown.Link: return [.link(destination: restore(l.destination ?? ""), children: inlines(l))]262 case let i as Markdown.Image: return [.image(source: restore(i.source ?? ""), alt: restore(i.plainText))]263 case is LineBreak: return [.lineBreak]264 case is SoftBreak: return [.softBreak]265 case let h as InlineHTML: return [.text(restore(h.rawHTML))]266 case let s as SymbolLink: return [.code(s.destination ?? "")]267 default: return splitText(restore(node.format()))268 }269 }270271 mutating func block(_ node: Markup) -> [MDBlock] {272 switch node {273 case let p as Paragraph:274 // display math inside a paragraph becomes its own block275 let all = inlines(p); var out: [MDBlock] = []; var run: [MDInline] = []276 func trimmed(_ r: [MDInline]) -> [MDInline] {277 var r = r278 while let f = r.first, f == .softBreak || f == .lineBreak { r.removeFirst() }279 while let l = r.last, l == .softBreak || l == .lineBreak { r.removeLast() }280 return r281 }282 for i in all {283 if case .math(let tex, true) = i { let t = trimmed(run); if !t.isEmpty { out.append(.paragraph(t)) }; run = []; out.append(.math(tex)) } else { run.append(i) }284 }285 let t = trimmed(run); if !t.isEmpty { out.append(.paragraph(t)) }286 return out287 case let h as Heading: return [.heading(level: h.level, inlines(h))]288 case let c as CodeBlock:289 var code = restore(c.code); if code.hasSuffix("\n") { code.removeLast() }290 let lang = c.language?.split(separator: " ").first.map(String.init)291 return [.code(language: lang?.isEmpty == false ? lang : nil, text: code)]292 case let q as BlockQuote: var out: [MDBlock] = []; for c in q.children { out += block(c) }; return [.quote(out)]293 case let l as UnorderedList: return [.list(ordered: false, start: 1, items: items(l))]294 case let l as OrderedList: return [.list(ordered: true, start: Int(l.startIndex), items: items(l))]295 case let t as Markdown.Table:296 let aligns: [MDTable.Alignment] = t.columnAlignments.map { switch $0 { case .center: .center; case .right: .trailing; default: .leading } }297 var header: [[MDInline]] = []; for cell in t.head.cells { header.append(inlines(cell)) }298 var rows: [[[MDInline]]] = []299 for row in t.body.rows { var r: [[MDInline]] = []; for cell in row.cells { r.append(inlines(cell)) }; rows.append(r) }300 return [.table(MDTable(alignments: aligns, header: header, rows: rows))]301 case is ThematicBreak: return [.rule]302 case let h as HTMLBlock: return [.html(restore(h.rawHTML).trimmingCharacters(in: .newlines))]303 default:304 let text = restore(node.format()).trimmingCharacters(in: .whitespacesAndNewlines)305 return text.isEmpty ? [] : [.paragraph([.text(text)])]306 }307 }308309 mutating func items(_ list: Markup) -> [MDListItem] {310 var out: [MDListItem] = []311 for child in list.children {312 guard let li = child as? ListItem else { continue }313 var blocks: [MDBlock] = []; for c in li.children { blocks += block(c) }314 out.append(MDListItem(checkbox: li.checkbox.map { $0 == .checked }, blocks: blocks))315 }316 return out317 }318 }319}320321/// Which links may be opened from model output. Anything else (javascript:, file:, custom schemes) is inert text.322public enum LinkPolicy {323 public static func safeURL(_ destination: String) -> URL? {324 guard let url = URL(string: destination.trimmingCharacters(in: .whitespaces)), let scheme = url.scheme?.lowercased() else { return nil }325 guard ["http", "https", "mailto"].contains(scheme) else { return nil }326 if scheme != "mailto", (url.host ?? "").isEmpty { return nil }327 return url328 }329}330