Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocFormats/MarkdownImporter.swift3// Description: Markdown import: CommonMark core blocks, GFM tables / strikethrough / task lists, footnotes, YAML front matter,4// LaTeX math, pandoc citations. A hand-written parser with bounded nesting; nothing is fetched or executed.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import DocModel10import DocContainer1112public enum MarkdownImporter {13 /// Never throws: an oversized input yields an empty document and a `.dropped` warning. Use `read` to get the error instead.14 /// `resourceLoader` receives the relative path of a linked image and returns its bytes; remote URLs are never fetched.15 public static func importDocument(_ text: String, resourceLoader: ((String) -> Data?)? = nil) -> ImportResult {16 do { return try read(text, resourceLoader: resourceLoader) } catch { return ImportFailure.result(error, format: .markdown) }17 }1819 public static func read(_ text: String, resourceLoader: ((String) -> Data?)? = nil) throws -> ImportResult {20 guard text.utf8.count <= FormatLimits.maxTextBytes else { throw FormatError.tooLarge("the Markdown source is larger than \(FormatLimits.maxTextBytes) bytes") }21 // The block parser is recursive (bounded by FormatLimits.maxMarkdownNesting); it runs on a stack sized for that bound,22 // whatever the size of the caller's stack.23 return DeepStack.run {24 let parser = MarkdownParser(loader: resourceLoader)25 return parser.run(text.hasPrefix("\u{FEFF}") ? String(text.dropFirst()) : text)26 }27 }28}2930final class MarkdownParser {31 struct Context {32 var depth = 033 var quote = 034 var list: (id: String, level: Int)?35 var inNote = false36 }3738 var warnings = WarningCollector()39 var images: ImageResolver40 var lists = ListAllocator()41 var references: [String: String] = [:]42 var noteLabels: Set<String> = []43 private var noteBodies: [String: [Paragraph]] = [:]44 private var meta = DocumentMetadata()4546 init(loader: ((String) -> Data?)?) { images = ImageResolver(loader: loader) }4748 static func normalizeLabel(_ s: String) -> String {49 s.lowercased().split(whereSeparator: { $0.isWhitespace }).joined(separator: " ")50 }5152 // MARK: Entry point5354 func run(_ source: String) -> ImportResult {55 var lines = splitLines(source)56 lines = frontMatter(lines)57 prescan(lines)58 var body = parseBlocks(lines, Context())59 var notes: [String: Note] = [:]60 resolveNotes(&body, ¬es)61 linkHeadingAnchors(&body)62 var doc = DocumentModel(meta: meta, body: body)63 doc.lists = lists.definitions64 doc.notes = notes65 doc.assets = images.assets66 return ImportResult(document: doc, warnings: warnings.warnings)67 }6869 /// Splits into lines, normalising line endings and removing characters the model cannot hold.70 private func splitLines(_ source: String) -> [String] {71 var lines: [String] = []72 var line = String.UnicodeScalarView()73 var previousCR = false74 var removed = false75 for u in source.unicodeScalars {76 switch u.value {77 case 0x0D: lines.append(String(line)); line = String.UnicodeScalarView(); previousCR = true; continue78 case 0x0A: if !previousCR { lines.append(String(line)); line = String.UnicodeScalarView() }79 case 0x09: line.append(u)80 case 0..<0x20, 0x7F: removed = true81 case 0xFFFC: line.append("\u{FFFD}")82 case 0x2028, 0x2029: lines.append(String(line)); line = String.UnicodeScalarView()83 default: line.append(u)84 }85 previousCR = false86 }87 lines.append(String(line))88 if removed { warnings.add(.dropped, "Control characters", "Control characters that cannot appear in a document were removed.") }89 return lines90 }9192 // MARK: Line helpers9394 private func isBlank(_ l: String) -> Bool { l.utf8.allSatisfy { $0 == 0x20 || $0 == 0x09 } }9596 /// Width of the leading white space in columns (tab stops every four columns).97 private func indent(_ l: String) -> Int {98 var col = 099 for b in l.utf8 {100 if b == 0x20 { col += 1 } else if b == 0x09 { col += 4 - col % 4 } else { break }101 }102 return col103 }104105 /// Removes up to `columns` columns of leading white space.106 private func strip(_ l: String, columns: Int) -> String {107 var col = 0108 var index = l.startIndex109 while index < l.endIndex, col < columns {110 let ch = l[index]111 if ch == " " { col += 1 } else if ch == "\t" {112 let width = 4 - col % 4113 if col + width > columns { return String(repeating: " ", count: col + width - columns) + l[l.index(after: index)...] }114 col += width115 } else { break }116 index = l.index(after: index)117 }118 return String(l[index...])119 }120121 private func trimLeading(_ l: String) -> String { String(l.drop { $0 == " " || $0 == "\t" }) }122 private func trimmed(_ l: String) -> String { l.trimmingCharacters(in: CharacterSet(charactersIn: " \t")) }123124 private func atxHeading(_ l: String) -> (level: Int, text: String)? {125 let t = trimLeading(l)126 var level = 0127 for ch in t { if ch == "#" { level += 1 } else { break } }128 guard (1...6).contains(level) else { return nil }129 let rest = t.dropFirst(level)130 guard rest.isEmpty || rest.first == " " || rest.first == "\t" else { return nil }131 var text = trimmed(String(rest))132 // Optional closing sequence of #.133 if let lastSpace = text.lastIndex(of: " "), text[text.index(after: lastSpace)...].allSatisfy({ $0 == "#" }), text.last == "#" {134 text = trimmed(String(text[..<lastSpace]))135 } else if !text.isEmpty, text.allSatisfy({ $0 == "#" }) { text = "" }136 return (level, text)137 }138139 private func isThematicBreak(_ l: String) -> Bool {140 var marker: Character?141 var count = 0142 for ch in l {143 if ch == " " || ch == "\t" { continue }144 guard ch == "-" || ch == "*" || ch == "_" else { return false }145 if let m = marker, m != ch { return false }146 marker = ch; count += 1147 }148 return count >= 3149 }150151 private func setextLevel(_ l: String) -> Int? {152 let t = trimmed(l)153 guard !t.isEmpty, indent(l) < 4 else { return nil }154 if t.allSatisfy({ $0 == "=" }) { return 1 }155 if t.allSatisfy({ $0 == "-" }) { return 2 }156 return nil157 }158159 private func fenceStart(_ l: String) -> (char: Character, count: Int, indent: Int, info: String)? {160 let t = trimLeading(l)161 guard let c = t.first, c == "`" || c == "~" else { return nil }162 let count = t.prefix { $0 == c }.count163 guard count >= 3 else { return nil }164 let info = trimmed(String(t.dropFirst(count)))165 if c == "`", info.contains("`") { return nil }166 return (c, count, indent(l), info)167 }168169 private func quoteStrip(_ l: String) -> String? {170 guard indent(l) < 4 else { return nil }171 let t = trimLeading(l)172 guard t.first == ">" else { return nil }173 let rest = t.dropFirst()174 return rest.first == " " ? String(rest.dropFirst()) : String(rest)175 }176177 struct Marker {178 var ordered: Bool179 var number: Int180 var delimiter: Character181 var contentOffset: Int182 var rest: String183 }184185 private func listMarker(_ l: String) -> Marker? {186 let ind = indent(l)187 guard ind < 4 else { return nil }188 let t = trimLeading(l)189 guard let first = t.first else { return nil }190 var markerLength = 0191 var ordered = false, number = 1192 var delimiter = first193 if first == "-" || first == "+" || first == "*" {194 markerLength = 1195 } else if first.isASCII, first.isNumber {196 let digits = t.prefix { $0.isASCII && $0.isNumber }197 guard digits.count <= 9, let n = Int(digits) else { return nil }198 let after = t.dropFirst(digits.count)199 guard let d = after.first, d == "." || d == ")" else { return nil }200 ordered = true; number = n; delimiter = d; markerLength = digits.count + 1201 } else { return nil }202 let after = String(t.dropFirst(markerLength))203 guard after.isEmpty || after.first == " " || after.first == "\t" else { return nil }204 if isBlank(after) { return Marker(ordered: ordered, number: number, delimiter: delimiter, contentOffset: ind + markerLength + 1, rest: "") }205 let spaces = indent(after)206 if spaces >= 5 {207 return Marker(ordered: ordered, number: number, delimiter: delimiter, contentOffset: ind + markerLength + 1, rest: strip(after, columns: 1))208 }209 return Marker(ordered: ordered, number: number, delimiter: delimiter, contentOffset: ind + markerLength + spaces, rest: trimLeading(after))210 }211212 private func sameList(_ a: Marker, _ b: Marker) -> Bool { a.ordered == b.ordered && a.delimiter == b.delimiter }213214 private func tableDelimiter(_ l: String) -> [TextAlignment?]? {215 guard l.contains("-"), indent(l) < 4 else { return nil }216 let cells = splitRow(l)217 guard !cells.isEmpty, cells.count <= FormatLimits.maxTableColumns else { return nil }218 var out: [TextAlignment?] = []219 for cell in cells {220 let c = trimmed(cell)221 let left = c.hasPrefix(":"), right = c.hasSuffix(":") && c.count > 1222 let dashes = c.dropFirst(left ? 1 : 0).dropLast(right ? 1 : 0)223 guard !dashes.isEmpty, dashes.allSatisfy({ $0 == "-" }) else { return nil }224 out.append(left && right ? .center : right ? .right : left ? .left : nil)225 }226 return out227 }228229 private func splitRow(_ l: String) -> [String] {230 var t = trimmed(l)231 if t.hasPrefix("|") { t.removeFirst() }232 var cells: [String] = []233 var cell = ""234 var escaped = false235 for ch in t {236 if escaped { cell.append(ch); escaped = false; continue }237 if ch == "\\" { cell.append(ch); escaped = true; continue }238 if ch == "|" { cells.append(cell); cell = ""; continue }239 cell.append(ch)240 }241 if !isBlank(cell) || cells.isEmpty { cells.append(cell) }242 return cells243 }244245 /// The dashed line of a pandoc simple table ("------ -------"): the character ranges of its columns.246 private func simpleTableRule(_ l: String) -> [Range<Int>]? {247 guard indent(l) < 4 else { return nil }248 var groups: [Range<Int>] = []249 var start: Int?250 var column = 0251 for ch in l {252 if ch == "-" { if start == nil { start = column } }253 else if ch == " " { if let s = start { groups.append(s..<column); start = nil } }254 else { return nil }255 column += 1256 }257 if let s = start { groups.append(s..<column) }258 guard groups.count >= 2, groups.count <= FormatLimits.maxTableColumns, groups.allSatisfy({ $0.count >= 2 }) else { return nil }259 return groups260 }261262 /// A pandoc simple table starting at `lines[i]` (its header line): header, dashed rule, rows up to a blank line.263 private func simpleTable(_ lines: [String], at i: Int, _ ctx: Context) -> (table: Table, next: Int)? {264 guard i + 1 < lines.count, !isBlank(lines[i]), indent(lines[i]) < 4, let columns = simpleTableRule(lines[i + 1]) else { return nil }265 func cells(_ line: String) -> [String] {266 let chars = Array(line)267 return columns.enumerated().map { k, range in268 let lower = k == 0 ? 0 : range.lowerBound269 let upper = k + 1 < columns.count ? columns[k + 1].lowerBound : chars.count270 guard lower < chars.count else { return "" }271 return trimmed(String(chars[lower..<min(chars.count, max(lower, upper))]))272 }273 }274 // Alignment follows the position of the header text over its dashes.275 let header = Array(lines[i])276 var alignments: [TextAlignment?] = []277 for range in columns {278 let slice = range.lowerBound < header.count ? Array(header[range.lowerBound..<min(header.count, range.upperBound)]) : []279 let leftFlush = slice.first.map { $0 != " " } ?? false280 let rightFlush = slice.count == range.count && (slice.last.map { $0 != " " } ?? false)281 alignments.append(leftFlush && rightFlush ? nil : rightFlush ? .right : leftFlush ? .left : slice.contains { $0 != " " } ? .center : nil)282 }283 var rows = [cells(lines[i])]284 var j = i + 2285 while j < lines.count, !isBlank(lines[j]), rows.count * columns.count < FormatLimits.maxTableCells {286 if simpleTableRule(lines[j]) != nil || trimmed(lines[j]).allSatisfy({ $0 == "-" }) { j += 1; break } // closing rule287 rows.append(cells(lines[j])); j += 1288 }289 return (table(rows, alignments, ctx), j)290 }291292 private func linkReference(_ l: String) -> (label: String, url: String)? {293 guard indent(l) < 4 else { return nil }294 let t = trimLeading(l)295 guard t.hasPrefix("["), !t.hasPrefix("[^"), let close = t.range(of: "]:") else { return nil }296 let label = String(t[t.index(after: t.startIndex)..<close.lowerBound])297 guard !label.isEmpty, !label.contains("["), !label.contains("]") else { return nil }298 var rest = trimmed(String(t[close.upperBound...]))299 guard !rest.isEmpty else { return nil }300 var url: String301 if rest.hasPrefix("<"), let end = rest.firstIndex(of: ">") {302 url = String(rest[rest.index(after: rest.startIndex)..<end]); rest = trimmed(String(rest[rest.index(after: end)...]))303 } else {304 url = String(rest.prefix { $0 != " " && $0 != "\t" }); rest = trimmed(String(rest.dropFirst(url.count)))305 }306 if !rest.isEmpty {307 guard let q = rest.first, let e = rest.last, (q == "\"" && e == "\"") || (q == "'" && e == "'") || (q == "(" && e == ")"), rest.count >= 2 else { return nil }308 }309 return (label, url)310 }311312 private func footnoteDefinition(_ l: String) -> (label: String, rest: String)? {313 guard indent(l) < 4 else { return nil }314 let t = trimLeading(l)315 guard t.hasPrefix("[^"), let close = t.range(of: "]:") else { return nil }316 let label = String(t[t.index(t.startIndex, offsetBy: 2)..<close.lowerBound])317 guard !label.isEmpty, !label.contains(where: { $0.isWhitespace || $0 == "[" || $0 == "]" }) else { return nil }318 return (label, trimLeading(String(t[close.upperBound...])))319 }320321 private func mathBlock(_ lines: [String], at i: Int) -> (latex: String, next: Int)? {322 let t = trimmed(lines[i])323 guard t.hasPrefix("$$"), indent(lines[i]) < 4 else { return nil }324 let rest = String(t.dropFirst(2))325 if rest.contains("$$") {326 guard rest.hasSuffix("$$") else { return nil }327 let latex = trimmed(String(rest.dropLast(2)))328 return latex.isEmpty ? nil : (latex, i + 1)329 }330 var body = [rest]331 var j = i + 1332 while j < lines.count, j - i < 2000 {333 if let end = lines[j].range(of: "$$") {334 guard isBlank(String(lines[j][end.upperBound...])) else { return nil }335 body.append(String(lines[j][..<end.lowerBound]))336 let latex = body.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines)337 return latex.isEmpty ? nil : (latex, j + 1)338 }339 body.append(lines[j]); j += 1340 }341 return nil342 }343344 /// True when `lines[i]` starts a block that interrupts a paragraph.345 private func startsBlock(_ lines: [String], at i: Int) -> Bool {346 let l = lines[i]347 if indent(l) >= 4 { return false }348 if i + 1 < lines.count, !isBlank(l), simpleTableRule(lines[i + 1]) != nil, simpleTableRule(l) == nil { return true } // header line of a pandoc simple table349 if atxHeading(l) != nil || fenceStart(l) != nil || quoteStrip(l) != nil || isThematicBreak(l) { return true }350 if let m = listMarker(l), !m.rest.isEmpty, !m.ordered || m.number == 1 { return true }351 let t = trimLeading(l)352 if t.hasPrefix("$$") || t.hasPrefix(":::") { return true }353 if l.contains("|"), i + 1 < lines.count, let d = tableDelimiter(lines[i + 1]), d.count == splitRow(l).count { return true }354 return false355 }356357 // MARK: Front matter and pre-scan358359 private func frontMatter(_ lines: [String]) -> [String] {360 guard let first = lines.first, trimmed(first) == "---", lines.count > 2 else { return lines }361 guard let end = lines.dropFirst().firstIndex(where: { let t = trimmed($0); return t == "---" || t == "..." }), end > 1 || lines.count > 2 else { return lines }362 let yaml = Array(lines[1..<end])363 // A "---" paragraph break followed by text and another rule is not front matter: require key: value on the first content line.364 guard let firstContent = yaml.first(where: { !isBlank($0) }), firstContent.contains(":"), !firstContent.hasPrefix(" ") else { return lines }365 var ignored: [String] = []366 var i = 0367 func unquote(_ v: String) -> String {368 let t = trimmed(v)369 if t.count >= 2, let f = t.first, let l = t.last, (f == "\"" && l == "\"") || (f == "'" && l == "'") {370 let inner = String(t.dropFirst().dropLast())371 return f == "\"" ? inner.replacingOccurrences(of: "\\\"", with: "\"").replacingOccurrences(of: "\\\\", with: "\\") : inner.replacingOccurrences(of: "''", with: "'")372 }373 return t374 }375 while i < yaml.count {376 let line = yaml[i]377 i += 1378 guard !isBlank(line), !line.hasPrefix("#"), let colon = line.firstIndex(of: ":"), !line.hasPrefix(" ") else { continue }379 let key = trimmed(String(line[..<colon])).lowercased()380 var value = trimmed(String(line[line.index(after: colon)...]))381 var items: [String] = []382 if value.hasPrefix("["), value.hasSuffix("]") {383 items = value.dropFirst().dropLast().split(separator: ",").map { unquote(String($0)) }.filter { !$0.isEmpty }384 value = items.joined(separator: ", ")385 } else if value.isEmpty || value == "|" || value == ">" || value == "|-" || value == ">-" {386 var block: [String] = []387 while i < yaml.count, yaml[i].hasPrefix(" ") || yaml[i].hasPrefix("-") || isBlank(yaml[i]) {388 let t = trimmed(yaml[i])389 if t.hasPrefix("- ") { items.append(unquote(String(t.dropFirst(2)))) } else if !t.isEmpty { block.append(t) }390 i += 1391 }392 value = items.isEmpty ? block.joined(separator: " ") : items.joined(separator: ", ")393 } else {394 value = unquote(value)395 items = [value]396 }397 value = TextSanitizer.clean(value)398 switch key {399 case "title": meta.title = value400 case "author", "authors": meta.author = value401 case "keywords", "tags": meta.keywords = items.isEmpty ? value.split(separator: ",").map { trimmed(String($0)) }.filter { !$0.isEmpty } : items.count == 1 ? items[0].split(separator: ",").map { trimmed(String($0)) }.filter { !$0.isEmpty } : items402 case "lang", "language": if !value.isEmpty { meta.language = value }403 case "subject", "description": meta.subject = value404 default: ignored.append(key)405 }406 }407 if !ignored.isEmpty {408 warnings.add(.dropped, "Front matter", "Front matter keys other than title, author, keywords, lang and subject are not kept: " + ignored.prefix(8).joined(separator: ", ") + ".")409 }410 return Array(lines[(end + 1)...])411 }412413 /// Collects link reference definitions and footnote labels, which may be used before they are defined.414 private func prescan(_ lines: [String]) {415 var fence: (Character, Int)?416 for raw in lines {417 var l = raw418 var guardCount = 0419 while let q = quoteStrip(l), guardCount < 64 { l = q; guardCount += 1 }420 if let f = fence {421 if let close = fenceStart(l), close.char == f.0, close.count >= f.1, close.info.isEmpty { fence = nil }422 continue423 }424 if let f = fenceStart(l) { fence = (f.char, f.count); continue }425 if let note = footnoteDefinition(l) { noteLabels.insert(note.label) }426 else if let ref = linkReference(l) {427 let key = MarkdownParser.normalizeLabel(ref.label)428 if references[key] == nil { references[key] = ref.url }429 }430 }431 }432433 // MARK: Blocks434435 private func inline(_ text: String) -> [Run] {436 var runs = MarkdownInlineParser(text, host: self).parse()437 // Trim white space at both ends of the paragraph (code spans keep theirs).438 func isText(_ r: Run) -> Bool { r.inline == nil && r.charStyle != StyleID.inlineCode }439 var lo = 0, hi = runs.count440 while lo < hi, isText(runs[lo]) {441 let t = String(runs[lo].text.drop { $0 == " " || $0 == "\t" })442 if t.isEmpty { lo += 1 } else { runs[lo].text = t; break }443 }444 while hi > lo, isText(runs[hi - 1]) {445 var t = runs[hi - 1].text446 while let c = t.last, c == " " || c == "\t" { t.removeLast() }447 if t.isEmpty { hi -= 1 } else { runs[hi - 1].text = t; break }448 }449 if lo > 0 || hi < runs.count { runs = Array(runs[lo..<hi]) }450 return runs451 }452453 private func paragraph(_ runs: [Run], style: String = StyleID.normal, _ ctx: Context) -> Paragraph {454 var p = Paragraph(style: style, runs: runs)455 if ctx.quote > 0, style == StyleID.normal {456 p.style = StyleID.quote457 if ctx.quote > 1 { p.props.indentLeft = 28 * Double(min(ctx.quote, 12)) }458 } else if ctx.inNote, style == StyleID.normal {459 p.style = StyleID.footnoteText460 }461 p.healRunBoundaries()462 return p463 }464465 private func codeParagraph(_ lines: [String], _ ctx: Context) -> Paragraph {466 let text = lines.joined(separator: lineSeparatorCharacter)467 return Paragraph(style: StyleID.code, runs: text.isEmpty ? [] : [Run(text)])468 }469470 func parseBlocks(_ lines: [String], _ ctx: Context) -> [Block] {471 var out: [Block] = []472 var i = 0473 while i < lines.count {474 let line = lines[i]475 if isBlank(line) { i += 1; continue }476477 if indent(line) >= 4 {478 var code: [String] = []479 while i < lines.count, isBlank(lines[i]) || indent(lines[i]) >= 4 { code.append(strip(lines[i], columns: 4)); i += 1 }480 while let last = code.last, isBlank(last) { code.removeLast() }481 out.append(.paragraph(codeParagraph(code, ctx)))482 continue483 }484 if let fence = fenceStart(line) {485 var code: [String] = []486 i += 1487 while i < lines.count {488 if let close = fenceStart(lines[i]), close.char == fence.char, close.count >= fence.count, close.info.isEmpty, indent(lines[i]) < 4 { i += 1; break }489 code.append(strip(lines[i], columns: fence.indent)); i += 1490 }491 if !fence.info.isEmpty { warnings.add(.dropped, "Code block language", "The language label of fenced code blocks is not kept.") }492 out.append(.paragraph(codeParagraph(code, ctx)))493 continue494 }495 if let math = mathBlock(lines, at: i) {496 out.append(.paragraph(paragraph([Run(inline: .equation(Equation(latex: math.latex, display: true)))], ctx)))497 i = math.next498 continue499 }500 if let heading = atxHeading(line) {501 out.append(.paragraph(headingParagraph(heading.text, level: heading.level, ctx)))502 i += 1503 continue504 }505 if simpleTableRule(line) == nil, let simple = simpleTable(lines, at: i, ctx) {506 out.append(.table(simple.table))507 i = simple.next508 continue509 }510 if isThematicBreak(line) {511 out.append(.paragraph(ModelIdiom.rule()))512 i += 1513 continue514 }515 if quoteStrip(line) != nil {516 var inner: [String] = []517 while i < lines.count {518 if let stripped = quoteStrip(lines[i]) { inner.append(stripped); i += 1 }519 else if !isBlank(lines[i]), let last = inner.last, !isBlank(last), !startsBlock(lines, at: i), listMarker(lines[i]) == nil { inner.append(lines[i]); i += 1 }520 else { break }521 }522 if ctx.depth >= FormatLimits.maxMarkdownNesting {523 warnings.add(.approximated, "Deep nesting", "Blocks nested deeper than \(FormatLimits.maxMarkdownNesting) levels are kept as plain text.")524 out.append(.paragraph(paragraph([Run(inner.joined(separator: " "))], ctx)))525 continue526 }527 var inside = ctx528 inside.depth += 1; inside.quote += 1529 let blocks = parseBlocks(inner, inside)530 if blocks.contains(where: { $0.paragraph?.style != StyleID.quote }) {531 warnings.add(.approximated, "Block quotes", "Lists, code, headings and tables inside block quotes are kept without the quote formatting.")532 }533 if inside.quote > 1 { warnings.add(.approximated, "Block quotes", "Nested block quotes are written as indented quotes.") }534 out.append(contentsOf: blocks)535 continue536 }537 if let first = listMarker(line) {538 out.append(contentsOf: list(lines, &i, first, ctx))539 continue540 }541 if let note = footnoteDefinition(line) {542 var content = [note.rest]543 i += 1544 while i < lines.count {545 let l = lines[i]546 if isBlank(l) {547 var j = i548 while j < lines.count, isBlank(lines[j]) { j += 1 }549 if j < lines.count, indent(lines[j]) >= 4 { content.append(contentsOf: Array(repeating: "", count: j - i)); i = j; continue }550 break551 }552 if indent(l) >= 4 { content.append(strip(l, columns: 4)); i += 1; continue }553 if let last = content.last, !isBlank(last), !startsBlock(lines, at: i), listMarker(l) == nil, footnoteDefinition(l) == nil, linkReference(l) == nil { content.append(trimLeading(l)); i += 1; continue }554 break555 }556 var inside = Context()557 inside.depth = ctx.depth + 1; inside.inNote = true558 let blocks = ctx.depth >= FormatLimits.maxMarkdownNesting ? [Block.paragraph(Paragraph(content.joined(separator: " "), style: StyleID.footnoteText))] : parseBlocks(content, inside)559 if noteBodies[note.label] == nil { noteBodies[note.label] = flattenToParagraphs(blocks, feature: "Footnotes") }560 continue561 }562 if linkReference(line) != nil { i += 1; continue }563 let lead = trimLeading(line)564 if lead.hasPrefix(":::") {565 // pandoc fenced div. "::: {#refs}" marks where the bibliography goes; other divs are transparent.566 if lead.contains("#refs") {567 var j = i + 1568 while j < lines.count, !trimLeading(lines[j]).hasPrefix(":::") { j += 1 }569 out.append(.bibliography(BibliographyBlock(title: nil)))570 i = min(lines.count, j + 1)571 } else {572 if lead.contains(where: { $0.isLetter }) { warnings.add(.dropped, "Fenced divs", "pandoc fenced div attributes are not kept; their content is.") }573 i += 1574 }575 continue576 }577 if lead.hasPrefix("<!--") {578 var j = i579 var body = ""580 while j < lines.count {581 body += lines[j]582 if lines[j].contains("-->") { break }583 j += 1584 }585 let inner = body.replacingOccurrences(of: "<!--", with: "").replacingOccurrences(of: "-->", with: "")586 if let close = lines[min(j, lines.count - 1)].range(of: "-->"), isBlank(String(lines[min(j, lines.count - 1)][close.upperBound...])) || j >= lines.count {587 if !isBlank(inner) { warnings.add(.dropped, "HTML comments", "HTML comments in the Markdown source were dropped.") }588 i = min(lines.count, j + 1)589 continue590 }591 }592 if line.contains("|"), i + 1 < lines.count, let alignments = tableDelimiter(lines[i + 1]) {593 let header = splitRow(line)594 if header.count == alignments.count {595 var rows = [header]596 i += 2597 while i < lines.count, !isBlank(lines[i]), !startsBlock(lines, at: i), rows.count * header.count < FormatLimits.maxTableCells { rows.append(splitRow(lines[i])); i += 1 }598 out.append(.table(table(rows, alignments, ctx)))599 continue600 }601 }602 // Paragraph (or setext heading).603 var text = [trimLeading(line)]604 i += 1605 var heading: Int?606 while i < lines.count {607 let l = lines[i]608 if isBlank(l) { break }609 if let level = setextLevel(l) { heading = level; i += 1; break }610 if startsBlock(lines, at: i) { break }611 if footnoteDefinition(l) != nil { break }612 text.append(trimLeading(l)); i += 1613 }614 if let level = heading {615 out.append(.paragraph(headingParagraph(text.map(trimmed).joined(separator: " "), level: level, ctx)))616 } else {617 out.append(.paragraph(paragraph(inline(text.joined(separator: "\n")), ctx)))618 }619 }620 return out621 }622623 private func headingParagraph(_ raw: String, level: Int, _ ctx: Context) -> Paragraph {624 var text = raw625 var anchor: String?626 // pandoc header attributes: "Title {#id .class}".627 if text.hasSuffix("}"), let open = text.lastIndex(of: "{") {628 let attributes = text[text.index(after: open)...].dropLast()629 if let id = attributes.split(separator: " ").first(where: { $0.hasPrefix("#") && $0.count > 1 }) { anchor = String(id.dropFirst()) }630 if anchor != nil || attributes.hasPrefix(".") || attributes == "-" { text = trimmed(String(text[..<open])) }631 }632 var runs = inline(text)633 if let anchor { runs.insert(Run(inline: .bookmark(TextSanitizer.clean(anchor))), at: 0) }634 var p = Paragraph(style: StyleID.heading(level), runs: runs)635 p.healRunBoundaries()636 return p637 }638639 private func list(_ lines: [String], _ i: inout Int, _ first: Marker, _ ctx: Context) -> [Block] {640 var out: [Block] = []641 var level = 0642 let listID: String643 if let parent = ctx.list {644 level = parent.level + 1645 if level > FormatLimits.maxListLevel {646 level = FormatLimits.maxListLevel647 warnings.add(.approximated, "Deep lists", "Lists nested deeper than \(FormatLimits.maxListLevel + 1) levels are flattened to the deepest level.")648 }649 listID = lists.sublist(parent: parent.id, level: level, ordered: first.ordered, start: first.number)650 } else {651 listID = lists.newList(ordered: first.ordered, start: first.number)652 }653 while i < lines.count, let marker = listMarker(lines[i]), sameList(marker, first), !(isThematicBreak(lines[i])) {654 var item = [marker.rest]655 i += 1656 while i < lines.count {657 let l = lines[i]658 if isBlank(l) {659 var j = i660 while j < lines.count, isBlank(lines[j]) { j += 1 }661 if j < lines.count, indent(lines[j]) >= marker.contentOffset {662 item.append(contentsOf: Array(repeating: "", count: j - i)); i = j663 continue664 }665 break666 }667 if indent(l) >= marker.contentOffset { item.append(strip(l, columns: marker.contentOffset)); i += 1; continue }668 if let last = item.last, !isBlank(last), listMarker(l) == nil, !startsBlock(lines, at: i) { item.append(trimLeading(l)); i += 1; continue }669 break670 }671 var j = i672 while j < lines.count, isBlank(lines[j]) { j += 1 }673 if j < lines.count, let next = listMarker(lines[j]), sameList(next, first), !isThematicBreak(lines[j]) { i = j }674 out.append(contentsOf: listItem(item, listID: listID, level: level, ctx))675 }676 return out677 }678679 private func listItem(_ item: [String], listID: String, level: Int, _ ctx: Context) -> [Block] {680 var lines = item681 // GFM task list items: the checkbox becomes a ballot box character.682 for (mark, box) in [("[ ] ", "☐ "), ("[x] ", "☑ "), ("[X] ", "☑ ")] where lines[0].hasPrefix(mark) {683 lines[0] = box + lines[0].dropFirst(mark.count)684 }685 if lines[0] == "[ ]" { lines[0] = "☐" } else if lines[0] == "[x]" || lines[0] == "[X]" { lines[0] = "☑" }686 var blocks: [Block]687 if ctx.depth >= FormatLimits.maxMarkdownNesting {688 warnings.add(.approximated, "Deep nesting", "Blocks nested deeper than \(FormatLimits.maxMarkdownNesting) levels are kept as plain text.")689 blocks = [.paragraph(Paragraph(lines.map(trimmed).filter { !$0.isEmpty }.joined(separator: " ")))]690 } else {691 var inside = ctx692 inside.depth += 1693 inside.list = (listID, level)694 blocks = parseBlocks(lines, inside)695 }696 let indentLeft = 36 + 18 * Double(level)697 let plain: Set<String> = [StyleID.normal, StyleID.quote, StyleID.footnoteText]698 if case .paragraph(var p)? = blocks.first, p.list == nil, plain.contains(p.style), !ModelIdiom.isRule(p) {699 p.list = ListItem(listID: listID, level: level)700 p.style = StyleID.listParagraph701 p.props.indentLeft = nil702 blocks[0] = .paragraph(p)703 } else {704 blocks.insert(.paragraph(Paragraph(style: StyleID.listParagraph, list: ListItem(listID: listID, level: level))), at: 0)705 }706 for k in blocks.indices.dropFirst() {707 if case .paragraph(var p) = blocks[k], p.list == nil, plain.contains(p.style), p.props.indentLeft == nil, !ModelIdiom.isRule(p) {708 p.style = StyleID.listParagraph709 p.props.indentLeft = indentLeft710 blocks[k] = .paragraph(p)711 }712 }713 return blocks714 }715716 private func table(_ rows: [[String]], _ alignments: [TextAlignment?], _ ctx: Context) -> Table {717 let columns = alignments.count718 var out: [TableRow] = []719 for (r, row) in rows.enumerated() {720 var cells: [TableCell] = []721 for c in 0..<columns {722 var p = Paragraph(style: StyleID.tableText, runs: c < row.count ? inline(row[c]) : [])723 if let a = alignments[c] { p.props.alignment = a }724 p.healRunBoundaries()725 cells.append(TableCell(paragraphs: [p]))726 }727 if row.count > columns { warnings.add(.dropped, "Table cells", "Cells beyond the number of header columns were dropped.") }728 out.append(TableRow(cells: cells, isHeader: r == 0))729 }730 return Table(columnWidths: Array(repeating: 468 / Double(columns), count: columns), rows: out)731 }732733 private func flattenToParagraphs(_ blocks: [Block], feature: String) -> [Paragraph] {734 var out: [Paragraph] = []735 for block in blocks {736 switch block {737 case .paragraph(let p): out.append(p)738 case .table(let t):739 warnings.add(.approximated, feature, "Tables inside \(feature.lowercased()) are written as tab-separated paragraphs.")740 for row in t.rows {741 var p = Paragraph(style: StyleID.footnoteText)742 for (k, cell) in row.cells.enumerated() {743 if k > 0 { p.runs.append(Run("\t")) }744 for q in cell.paragraphs { p.runs.append(contentsOf: q.runs) }745 }746 out.append(p)747 }748 default: break749 }750 }751 return out.isEmpty ? [Paragraph(style: StyleID.footnoteText)] : out752 }753754 // MARK: Post-processing755756 /// Replaces temporary note references by real notes; references without a definition become literal text again.757 /// Only definitions reachable from the body are kept, so no reference can point at a note that is not in the document.758 private func resolveNotes(_ body: inout [Block], _ notes: inout [String: Note]) {759 let reachable = PendingNotes.reachable(from: body, definitions: noteBodies)760 let ids = Dictionary(uniqueKeysWithValues: reachable.map { ($0, DocID.make()) })761 func fix(_ p: inout Paragraph) {762 for k in p.runs.indices {763 guard case .note(let ref)? = p.runs[k].inline, let pending = PendingNotes.parse(ref) else { continue }764 if let id = ids[pending.target] { p.runs[k].inline = .note(NoteRef(kind: .footnote, id: id)) }765 else { p.runs[k] = Run("[^\(pending.target)]") }766 }767 }768 DocumentModel.mutateParagraphs(in: &body, fix)769 for (label, id) in ids {770 var paragraphs = noteBodies[label] ?? []771 for k in paragraphs.indices { fix(¶graphs[k]) }772 notes[id] = Note(id: id, kind: .footnote, paragraphs: paragraphs)773 }774 let unused = noteBodies.count - reachable.count775 if unused > 0 { warnings.add(.dropped, "Footnotes", "\(unused) footnote definition\(unused == 1 ? " is" : "s are") never referenced and \(unused == 1 ? "was" : "were") dropped.") }776 }777778 /// Internal links ("#slug") that point at a heading get a bookmark on that heading, so they keep working.779 private func linkHeadingAnchors(_ body: inout [Block]) {780 var wanted: Set<String> = []781 var existing: Set<String> = []782 DocumentModel.forEachParagraph(in: body) { p, _ in783 for r in p.runs {784 if let link = r.props.link, link.hasPrefix("#"), link.count > 1 { wanted.insert(String(link.dropFirst())) }785 if case .bookmark(let name)? = r.inline { existing.insert(name) }786 }787 }788 wanted.subtract(existing)789 guard !wanted.isEmpty else { return }790 var slugger = Slugger()791 for k in body.indices {792 guard case .paragraph(var p) = body[k], p.style.hasPrefix("Heading") else { continue }793 let slug = slugger.unique(p.visibleText)794 if wanted.contains(slug) {795 p.runs.insert(Run(inline: .bookmark(slug)), at: 0)796 body[k] = .paragraph(p)797 wanted.remove(slug)798 }799 }800 }801}802