SPB Git

Repository Public

MarkdownModel.swift 16.8 KB 330 lines swift

1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Core/Markdown/MarkdownModel.swift3// Description: Render model for assistant text (blocks, inlines, math) and the Markdown → model parser with incremental chunking for streaming.4import Foundation5import Markdown67// MARK: - Render model89public indirect enum MDInline: Sendable, Hashable {10    case text(String)11    case emphasis([MDInline])12    case strong([MDInline])13    case strikethrough([MDInline])14    case code(String)15    case link(destination: String, children: [MDInline])16    case image(source: String, alt: String)17    case math(String, display: Bool)18    case lineBreak19    case softBreak2021    /// Text as a reader would copy it (math keeps its TeX source).22    public var plainText: String {23        switch self {24        case .text(let s), .code(let s): s25        case .emphasis(let c), .strong(let c), .strikethrough(let c): c.map(\.plainText).joined()26        case .link(_, let c): c.map(\.plainText).joined()27        case .image(_, let alt): alt28        case .math(let tex, let display): display ? "$$\(tex)$$" : "$\(tex)$"29        case .lineBreak: "\n"30        case .softBreak: " "31        }32    }33}3435public struct MDListItem: Sendable, Hashable {36    public var checkbox: Bool?37    public var blocks: [MDBlock]38}3940public struct MDTable: Sendable, Hashable {41    public enum Alignment: Sendable, Hashable { case leading, center, trailing }42    public var alignments: [Alignment]43    public var header: [[MDInline]]44    public var rows: [[[MDInline]]]45    public var columnCount: Int { max(header.count, rows.map(\.count).max() ?? 0) }46}4748public indirect enum MDBlock: Sendable, Hashable {49    case paragraph([MDInline])50    case heading(level: Int, [MDInline])51    case code(language: String?, text: String)52    case quote([MDBlock])53    case list(ordered: Bool, start: Int, items: [MDListItem])54    case table(MDTable)55    case rule56    case math(String)57    /// Raw HTML from the model is never interpreted: it is displayed as literal text.58    case html(String)5960    public var plainText: String {61        switch self {62        case .paragraph(let i), .heading(_, let i): i.map(\.plainText).joined()63        case .code(_, let t): t64        case .quote(let b): b.map(\.plainText).joined(separator: "\n")65        case .list(let ordered, let start, let items):66            items.enumerated().map { n, it in (ordered ? "\(start + n). " : "• ") + it.blocks.map(\.plainText).joined(separator: "\n") }.joined(separator: "\n")67        case .table(let t): ([t.header] + t.rows).map { $0.map { $0.map(\.plainText).joined() }.joined(separator: "\t") }.joined(separator: "\n")68        case .rule: "———"69        case .math(let tex): "$$\(tex)$$"70        case .html(let s): s71        }72    }73}7475/// One independently parsed slice of a message. While a message streams only its last chunk changes, so the views of76/// every earlier chunk are left untouched.77public struct MDChunk: Sendable, Hashable, Identifiable {78    public let id: Int79    public let source: String80    public let blocks: [MDBlock]81}8283// MARK: - Parser8485public final class MarkdownParser: @unchecked Sendable {86    public static let shared = MarkdownParser()87    private let lock = NSLock()88    private var cache: [String: [MDBlock]] = [:]89    private var order: [String] = []90    private let capacity: Int9192    public init(capacity: Int = 6000) { self.capacity = capacity }9394    /// Splits `source` into top-level chunks and parses each (cached by content).95    public func chunks(_ source: String) -> [MDChunk] {96        Self.split(source).enumerated().map { i, s in MDChunk(id: i, source: s, blocks: blocks(forChunk: s)) }97    }9899    /// Parses a whole document in one piece (exports, tests).100    public func parse(_ source: String) -> [MDBlock] { Self.parseUncached(source) }101102    private func blocks(forChunk s: String) -> [MDBlock] {103        if let hit = lock.withLock({ cache[s] }) { return hit }104        let parsed = Self.parseUncached(s)105        lock.withLock {106            if cache[s] == nil { order.append(s) }107            cache[s] = parsed108            if order.count > capacity { for k in order.prefix(capacity / 4) { cache[k] = nil }; order.removeFirst(capacity / 4) }109        }110        return parsed111    }112113    /// Cuts at blank lines that are outside fenced code and display math, and never between the lines of a list or a114    /// table (those keep their own blank-line semantics inside one chunk when they are contiguous).115    static func split(_ source: String) -> [String] {116        var chunks: [String] = []; var current: [Substring] = []117        var fence: (char: Character, count: Int)?; var inDisplayMath = false118        func flush() { let s = current.joined(separator: "\n"); if !s.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { chunks.append(s) }; current.removeAll(keepingCapacity: true) }119        for line in source.split(separator: "\n", omittingEmptySubsequences: false) {120            let trimmed = line.drop(while: { $0 == " " })121            if let f = fence {122                current.append(line)123                if trimmed.hasPrefix(String(repeating: f.char, count: f.count)), trimmed.allSatisfy({ $0 == f.char || $0 == " " }) { fence = nil }124                continue125            }126            if let c = trimmed.first, c == "`" || c == "~" {127                let n = trimmed.prefix(while: { $0 == c }).count128                if n >= 3, line.count - trimmed.count < 4 {129                    // a fence that follows a list item stays with it; otherwise it starts its own chunk130                    if let prev = current.last, prev.trimmingCharacters(in: .whitespaces).isEmpty { flush() }131                    fence = (c, n); current.append(line); continue132                }133            }134            let dollars = line.components(separatedBy: "$$").count - 1135            if dollars % 2 == 1 { inDisplayMath.toggle() }136            if trimmed.hasPrefix("\\[") && !line.contains("\\]") { inDisplayMath = true }137            if inDisplayMath && line.contains("\\]") && !trimmed.hasPrefix("\\[") { inDisplayMath = false }138            if line.allSatisfy({ $0 == " " || $0 == "\t" }) && !inDisplayMath { flush() } else { current.append(line) }139        }140        flush()141        return chunks142    }143144    // MARK: Math protection145146    private static let open: Character = "\u{E000}", close: Character = "\u{E001}"147148    struct Protected { var text: String; var math: [(tex: String, display: Bool, raw: String)] }149150    /// Replaces TeX spans by private-use placeholders so that the Markdown parser cannot mangle `_`, `*` or `\\` in them.151    /// Code spans and fenced blocks are skipped. `$…$` follows the usual rule: no space just inside the delimiters and152    /// no digit right after the closing one (so "$5 and $10" stays text).153    static func protectMath(_ source: String) -> Protected {154        var out = ""; out.reserveCapacity(source.count)155        var math: [(String, Bool, String)] = []156        let s = Array(source); var i = 0; let n = s.count157        var atLineStart = true158        func emit(_ tex: String, _ display: Bool, _ raw: String) { out.append(open); out.append(String(math.count)); out.append(close); math.append((tex.trimmingCharacters(in: .whitespacesAndNewlines), display, raw)) }159        func find(_ needle: [Character], from: Int, sameLine: Bool) -> Int? {160            var j = from161            while j + needle.count <= n {162                if sameLine && s[j] == "\n" { return nil }163                if s[j] == "\\" && needle != ["\\", ")"] && needle != ["\\", "]"] { j += 2; continue }164                if Array(s[j..<(j + needle.count)]) == needle { return j }165                j += 1166            }167            return nil168        }169        while i < n {170            let c = s[i]171            if atLineStart {172                // fenced code block: copy verbatim up to the closing fence173                var k = i; while k < n, s[k] == " " { k += 1 }174                if k < n, s[k] == "`" || s[k] == "~", k - i < 4 {175                    let fc = s[k]; var run = 0; while k + run < n, s[k + run] == fc { run += 1 }176                    if run >= 3 {177                        var j = k + run; var closed = false178                        while j < n {179                            guard let nl = s[j...].firstIndex(of: "\n") else { j = n; break }180                            var p = nl + 1; while p < n, s[p] == " " { p += 1 }181                            var r = 0; while p + r < n, s[p + r] == fc { r += 1 }182                            if r >= run { var e = p + r; while e < n, s[e] != "\n" { e += 1 }; j = e; closed = true; break }183                            j = nl + 1184                        }185                        _ = closed186                        out.append(contentsOf: s[i..<j]); i = j; atLineStart = false; continue187                    }188                }189            }190            atLineStart = c == "\n"191            if c == "`" { // inline code span: same-length backtick run closes it192                var run = 0; while i + run < n, s[i + run] == "`" { run += 1 }193                var j = i + run; var end: Int?194                while j < n { if s[j] == "`" { var r = 0; while j + r < n, s[j + r] == "`" { r += 1 }; if r == run { end = j + r; break }; j += r } else { j += 1 } }195                let stop = end ?? (i + run)196                out.append(contentsOf: s[i..<stop]); i = stop; continue197            }198            if c == "\\", i + 1 < n {199                if s[i + 1] == "[", let e = find(["\\", "]"], from: i + 2, sameLine: false) { emit(String(s[(i + 2)..<e]), true, String(s[i..<(e + 2)])); i = e + 2; continue }200                if s[i + 1] == "(", let e = find(["\\", ")"], from: i + 2, sameLine: false) { emit(String(s[(i + 2)..<e]), false, String(s[i..<(e + 2)])); i = e + 2; continue }201                out.append(c); out.append(s[i + 1]); i += 2; continue202            }203            if c == "$" {204                if i + 1 < n, s[i + 1] == "$" {205                    if let e = find(["$", "$"], from: i + 2, sameLine: false) { emit(String(s[(i + 2)..<e]), true, String(s[i..<(e + 2)])); i = e + 2; continue }206                } else if i + 1 < n, !s[i + 1].isWhitespace, let e = find(["$"], from: i + 1, sameLine: true), e > i + 1, !s[e - 1].isWhitespace, !(e + 1 < n && s[e + 1].isNumber), !(i > 0 && s[i - 1].isNumber),207                          !s[i + 1].isNumber || s[(i + 1)..<e].contains(where: { "\\^_{}=+*/<>".contains($0) }) || !s[(i + 1)..<e].contains(" ") {208                    emit(String(s[(i + 1)..<e]), false, String(s[i...e])); i = e + 1; continue209                }210            }211            out.append(c); i += 1212        }213        return Protected(text: out, math: math)214    }215216    // MARK: Conversion217218    static func parseUncached(_ source: String) -> [MDBlock] {219        let protected = protectMath(source)220        let doc = Document(parsing: protected.text, options: [.disableSmartOpts])221        var conv = Converter(math: protected.math)222        return doc.children.flatMap { conv.block($0) }223    }224225    private struct Converter {226        let math: [(tex: String, display: Bool, raw: String)]227228        /// Puts the original TeX source back (inside code, where placeholders must not survive).229        func restore(_ s: String) -> String {230            guard s.contains(MarkdownParser.open) else { return s }231            var out = ""; var rest = Substring(s)232            while let o = rest.firstIndex(of: MarkdownParser.open), let c = rest[o...].firstIndex(of: MarkdownParser.close) {233                out += rest[..<o]234                if let idx = Int(rest[rest.index(after: o)..<c]), math.indices.contains(idx) { out += math[idx].raw }235                rest = rest[rest.index(after: c)...]236            }237            return out + rest238        }239240        func splitText(_ s: String) -> [MDInline] {241            guard s.contains(MarkdownParser.open) else { return s.isEmpty ? [] : [.text(s)] }242            var out: [MDInline] = []; var rest = Substring(s)243            while let o = rest.firstIndex(of: MarkdownParser.open), let c = rest[o...].firstIndex(of: MarkdownParser.close) {244                if o > rest.startIndex { out.append(.text(String(rest[..<o]))) }245                if let idx = Int(rest[rest.index(after: o)..<c]), math.indices.contains(idx) { out.append(.math(math[idx].tex, display: math[idx].display)) }246                rest = rest[rest.index(after: c)...]247            }248            if !rest.isEmpty { out.append(.text(String(rest))) }249            return out250        }251252        func inlines(_ node: Markup) -> [MDInline] { node.children.flatMap { inline($0) } }253254        func inline(_ node: Markup) -> [MDInline] {255            switch node {256            case let t as Markdown.Text: return splitText(t.string)257            case let e as Emphasis: return [.emphasis(inlines(e))]258            case let s as Strong: return [.strong(inlines(s))]259            case let s as Strikethrough: return [.strikethrough(inlines(s))]260            case let c as InlineCode: return [.code(restore(c.code))]261            case let l as Markdown.Link: return [.link(destination: restore(l.destination ?? ""), children: inlines(l))]262            case let i as Markdown.Image: return [.image(source: restore(i.source ?? ""), alt: restore(i.plainText))]263            case is LineBreak: return [.lineBreak]264            case is SoftBreak: return [.softBreak]265            case let h as InlineHTML: return [.text(restore(h.rawHTML))]266            case let s as SymbolLink: return [.code(s.destination ?? "")]267            default: return splitText(restore(node.format()))268            }269        }270271        mutating func block(_ node: Markup) -> [MDBlock] {272            switch node {273            case let p as Paragraph:274                // display math inside a paragraph becomes its own block275                let all = inlines(p); var out: [MDBlock] = []; var run: [MDInline] = []276                func trimmed(_ r: [MDInline]) -> [MDInline] {277                    var r = r278                    while let f = r.first, f == .softBreak || f == .lineBreak { r.removeFirst() }279                    while let l = r.last, l == .softBreak || l == .lineBreak { r.removeLast() }280                    return r281                }282                for i in all {283                    if case .math(let tex, true) = i { let t = trimmed(run); if !t.isEmpty { out.append(.paragraph(t)) }; run = []; out.append(.math(tex)) } else { run.append(i) }284                }285                let t = trimmed(run); if !t.isEmpty { out.append(.paragraph(t)) }286                return out287            case let h as Heading: return [.heading(level: h.level, inlines(h))]288            case let c as CodeBlock:289                var code = restore(c.code); if code.hasSuffix("\n") { code.removeLast() }290                let lang = c.language?.split(separator: " ").first.map(String.init)291                return [.code(language: lang?.isEmpty == false ? lang : nil, text: code)]292            case let q as BlockQuote: var out: [MDBlock] = []; for c in q.children { out += block(c) }; return [.quote(out)]293            case let l as UnorderedList: return [.list(ordered: false, start: 1, items: items(l))]294            case let l as OrderedList: return [.list(ordered: true, start: Int(l.startIndex), items: items(l))]295            case let t as Markdown.Table:296                let aligns: [MDTable.Alignment] = t.columnAlignments.map { switch $0 { case .center: .center; case .right: .trailing; default: .leading } }297                var header: [[MDInline]] = []; for cell in t.head.cells { header.append(inlines(cell)) }298                var rows: [[[MDInline]]] = []299                for row in t.body.rows { var r: [[MDInline]] = []; for cell in row.cells { r.append(inlines(cell)) }; rows.append(r) }300                return [.table(MDTable(alignments: aligns, header: header, rows: rows))]301            case is ThematicBreak: return [.rule]302            case let h as HTMLBlock: return [.html(restore(h.rawHTML).trimmingCharacters(in: .newlines))]303            default:304                let text = restore(node.format()).trimmingCharacters(in: .whitespacesAndNewlines)305                return text.isEmpty ? [] : [.paragraph([.text(text)])]306            }307        }308309        mutating func items(_ list: Markup) -> [MDListItem] {310            var out: [MDListItem] = []311            for child in list.children {312                guard let li = child as? ListItem else { continue }313                var blocks: [MDBlock] = []; for c in li.children { blocks += block(c) }314                out.append(MDListItem(checkbox: li.checkbox.map { $0 == .checked }, blocks: blocks))315            }316            return out317        }318    }319}320321/// Which links may be opened from model output. Anything else (javascript:, file:, custom schemes) is inert text.322public enum LinkPolicy {323    public static func safeURL(_ destination: String) -> URL? {324        guard let url = URL(string: destination.trimmingCharacters(in: .whitespaces)), let scheme = url.scheme?.lowercased() else { return nil }325        guard ["http", "https", "mailto"].contains(scheme) else { return nil }326        if scheme != "mailto", (url.host ?? "").isEmpty { return nil }327        return url328    }329}330
Files main
Menu

Theme