Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Tests/DocFormatsTests/MarkdownTests.swift3// Description: Markdown import (blocks, inlines, GFM, footnotes, front matter, math, citations, images) and export (escaping,4// lists, tables, notes, options), asserted on the resulting model and on the exact text written.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Testing9import Foundation10import DocModel11import DocContainer12@testable import DocFormats1314/// A citation formatter for tests: "(KEY locator)" citations and one bibliography paragraph per cited key.15struct StubCitations: CitationFormatting {16 func citationText(_ citation: Citation, document: DocumentModel, numbering: Numbering) -> String {17 "(" + citation.items.map { $0.key.uppercased() + ($0.locator.map { " " + $0 } ?? "") }.joined(separator: "; ") + ")"18 }19 func bibliographyParagraphs(document: DocumentModel, numbering: Numbering) -> [Paragraph] {20 numbering.citedKeys.map { key in Paragraph(style: StyleID.bibliography, runs: [Run("[\(numbering.citationOrder[key] ?? 0)] "), Run(document.bibEntry(key)?.title ?? key, props: CharProps(italic: true))]) }21 }22}2324@Suite struct MarkdownImportTests {25 private func blocks(_ md: String) -> [String] { Describe.blocks(MarkdownImporter.importDocument(md).document) }26 private func line(_ md: String) -> String { blocks(md).joined(separator: " ¶ ") }2728 @Test func blocksAndInlines() throws {29 let png = Fixture.png()30 let md = """31 # Heading *one* {#top}3233 Para with **bold**, *italic*, ***both***, ~~strike~~, `code`, [link](http://example.org "t"), <u>under</u>, H~2~O, x^2^.34 Soft break here 35 hard break. Math $a^2+b^2$ and cite [@knuth1984, p. 4; -@doe:2020] note[^1].3637 Setext38 ------3940 - one41 - two42 1. nested a43 2. nested b4445 continuation of two46 - [ ] task47 - [x] done4849 3. three50 4. four5152 > quote line53 > more5455 ```swift56 let x = 157 indented58 ```5960 | A | B |61 |:--|--:|62 | 1 | 2 \\| 3 |6364 $$65 E = mc^266 $$6768 ---6970 )) and  [ref link][r] <http://auto.link>7172 [r]: http://ref.example "Title"73 [^1]: The note74 continues.7576 Second para.77 """78 let result = MarkdownImporter.importDocument(md)79 #expect(Describe.blocks(result.document) == [80 "Heading1: <bm top>Heading {i:one}",81 "Normal: Para with {b:bold}, {i:italic}, {bi:both}, {s:strike}, {c:code}, [link](http://example.org), {u:under}, H{_:2}O, x{^:2}. Soft break here⏎hard break. Math <eq a^2+b^2> and cite <cite |knuth1984|p. 4|;|-doe:2020||> note<footnote FootnoteText: The note continues. / FootnoteText: Second para.>.",82 "Heading2: Setext",83 "ListParagraph L0(•): one",84 "ListParagraph L0(•): two",85 "ListParagraph L1(1.): nested a",86 "ListParagraph L1(2.): nested b",87 "ListParagraph: continuation of two",88 "ListParagraph L0(•): ☐ task",89 "ListParagraph L0(•): ☑ done",90 "ListParagraph L0(3.): three",91 "ListParagraph L0(4.): four",92 "Quote: quote line more",93 "Code: let x = 1⏎ indented",94 "TABLE 2 cols",95 " H TableText: A | TableText: B",96 " R TableText: 1 | TableText: 2 | 3",97 "Normal: <EQ E = mc^2>",98 "RULE",99 "Normal: <img 6x5 alt=alt text data=\(png.count)> and remote [ref link](http://ref.example) [http://auto.link](http://auto.link)",100 ])101 let doc = result.document102 #expect(doc.assets.count == 1)103 #expect(doc.assets.values.first?.mimeType == "image/png")104 #expect(doc.notes.count == 1)105 #expect(doc.lists.count == 2)106 // Table alignment comes from the delimiter row.107 let table = try #require(doc.body.compactMap(\.table).first)108 #expect(table.rows[1].cells.map { $0.paragraphs[0].props.alignment } == [.left, .right])109 #expect(table.rows[0].isHeader && !table.rows[1].isHeader)110 // The hyperlink carries the character style, emphasis is direct formatting.111 let para = try #require(doc.body[1].paragraph)112 #expect(para.runs.first { $0.props.link != nil }?.charStyle == StyleID.hyperlink)113 #expect(para.runs.first { $0.text == "code" }?.charStyle == StyleID.inlineCode)114 // The continuation paragraph is indented under its item.115 #expect(doc.body[7].paragraph?.props.indentLeft == 36)116 #expect(result.warnings.map(\.feature).sorted() == ["Code block language", "Remote images"])117 #expect(result.warnings.first { $0.feature == "Remote images" }?.severity == .dropped)118 }119120 @Test func frontMatter() {121 let result = MarkdownImporter.importDocument("""122 ---123 title: "Tïtle: with \\"quotes\\""124 author:125 - Ada Lovelace126 - 'Charles O''Brien'127 keywords: [alpha, "beta gamma", delta]128 lang: fr-CA129 subject: A subject130 date: 2026-01-01131 custom: value132 ---133 Body.134 """)135 let meta = result.document.meta136 #expect(meta.title == "Tïtle: with \"quotes\"")137 #expect(meta.author == "Ada Lovelace, Charles O'Brien")138 #expect(meta.keywords == ["alpha", "beta gamma", "delta"])139 #expect(meta.language == "fr-CA")140 #expect(meta.subject == "A subject")141 #expect(Describe.blocks(result.document) == ["Normal: Body."])142 #expect(result.warnings.count == 1)143 #expect(result.warnings[0].detail.contains("date, custom"))144 // A rule at the top of a document without key: value content is a rule, not front matter.145 #expect(blocks("---\n\nText\n\n---\n") == ["RULE", "Normal: Text", "RULE"])146 #expect(blocks("---\njust text\n---\n").first == "RULE")147 // Comma separated keywords in a plain scalar are split too.148 #expect(MarkdownImporter.importDocument("---\nkeywords: a, b\n---\n").document.meta.keywords == ["a", "b"])149 }150151 @Test(arguments: [152 ("*a **b** c*", "Normal: {i:a }{bi:b}{i: c}"),153 ("**a*b*c**", "Normal: {b:a}{bi:b}{b:c}"),154 ("***a** b*", "Normal: {bi:a}{i: b}"),155 ("***a* b**", "Normal: {bi:a}{b: b}"),156 ("snake_case_word and 2 * 3 * 4", "Normal: snake_case_word and 2 * 3 * 4"),157 ("_em_ __strong__ ___both___", "Normal: {i:em} {b:strong} {bi:both}"),158 ("\\*not\\* \\_em\\_ \\# \\[x\\]", "Normal: *not* _em_ # [x]"),159 ("a~~b~~c ~~~~ ~single~", "Normal: a{s:b}c ~~~~ {_:single}"),160 ("`` code ` tick `` and `a b`", "Normal: {c:code ` tick} and {c:a b}"),161 ("unclosed `tick and *star and [bracket and **bold", "Normal: unclosed `tick and *star and [bracket and **bold"),162 ("& AB © &nosuch; � AT&T", "Normal: & AB © &nosuch; \u{FFFD} AT&T"),163 ("<http://x.y/a?b=c> and http://x.y/z. And (https://e.org/p).", "Normal: [http://x.y/a?b=c](http://x.y/a?b=c) and [http://x.y/z](http://x.y/z). And ([https://e.org/p](https://e.org/p))."),164 ("<me@example.org>", "Normal: [me@example.org](mailto:me@example.org)"),165 ("It costs $5 and $6, \\$x\\$, but $x$ and $ y $ no.", "Normal: It costs $5 and $6, $x$, but <eq x> and $ y $ no."),166 ("Inline $$x^2$$ display", "Normal: Inline <EQ x^2> display"),167 ("**bold [link *it*](u) end**", "Normal: {b:bold }{b:[link ](u)}{bi:[it](u)}{b: end}"),168 ("[outer [inner](i) text](o)", "Normal: [outer [inner](i) text](o)"),169 ("", "Normal: a b c"),170 ("line one\\\nline two", "Normal: line one⏎line two"),171 ("a<br>b<br/>c <mark>hi</mark> <sub>s</sub><sup>p</sup>", "Normal: a⏎b⏎c {h:hi} {_:s}{^:p}"),172 ("x <span class=\"y\">z</span>", "Normal: x <span class=\"y\">z</span>"),173 ("<a id=\"anchor\"></a>Text", "Normal: <bm anchor>Text"),174 ("[text][missing] and [shortcut] and []()", "Normal: [text][missing] and [shortcut] and [](<empty>)"),175 ("*\u{4E2D}\u{6587}* **\u{0627}\u{0644}\u{0639}\u{0631}\u{0628}\u{064A}\u{0629}**", "Normal: {i:\u{4E2D}\u{6587}} {b:\u{0627}\u{0644}\u{0639}\u{0631}\u{0628}\u{064A}\u{0629}}"),176 ])177 func inlineEdgeCases(source: String, expected: String) {178 let want = expected.replacingOccurrences(of: "[](<empty>)", with: "")179 let got = line(source)180 if expected.contains("<empty>") { #expect(got.hasPrefix("Normal: [text][missing] and [shortcut] and"), "\(got)") }181 else { #expect(got == want, "source: \(source)") }182 }183184 @Test func scriptLinksAreRemoved() {185 let result = MarkdownImporter.importDocument("[click](javascript:alert(1)) [data](data:text/html;base64,PHNjcmlwdD4=) [ok](https://ok.example) <javascript:alert(2)>")186 let p = result.document.body[0].paragraph!187 #expect(p.runs.compactMap(\.props.link) == ["https://ok.example"])188 #expect(p.visibleText == "click data ok javascript:alert(2)")189 #expect(result.warnings.contains { $0.feature == "Active content" && $0.severity == .dropped && $0.count == 2 })190 }191192 @Test func headings() {193 #expect(blocks("# One\n## Two ##\n###### Six\n####### Seven\n#NoSpace\n#\n") == [194 "Heading1: One", "Heading2: Two", "Heading6: Six", "Normal: ####### Seven #NoSpace", "Heading1: "])195 #expect(blocks("Title\n=====\n\nSub\ntitle\n---\n") == ["Heading1: Title", "Heading2: Sub title"])196 #expect(blocks("# With `code` and [link](u) {#my-id .unnumbered}") == ["Heading1: <bm my-id>With {c:code} and [link](u)"])197 #expect(blocks("# Braces {not an attribute}") == ["Heading1: Braces {not an attribute}"])198 }199200 @Test func lists() throws {201 let result = MarkdownImporter.importDocument("""202 1. one203 2. two204 - bullet a205 continues lazily206 - bullet b207 1. deep208 3. three209210 * other list211212 7) seven213 8) eight214215 continuation of eight216217 para218219 code after the list220 """)221 #expect(Describe.blocks(result.document) == [222 "ListParagraph L0(1.): one",223 "ListParagraph L0(2.): two",224 "ListParagraph L1(◦): bullet a continues lazily",225 "ListParagraph L1(◦): bullet b",226 "ListParagraph L2(1.): deep",227 "ListParagraph L0(3.): three",228 "ListParagraph L0(•): other list",229 "ListParagraph L0(7.): seven",230 "ListParagraph L0(8.): eight",231 "ListParagraph: continuation of eight", // indented at least to the item's content column: still part of the item232 "Normal: para",233 "Code: code after the list",234 ])235 let ids = result.document.body.compactMap { $0.paragraph?.list?.listID }236 #expect(Set(ids).count == 3) // three separate lists, numbering restarts237 #expect(ids[0] == ids[4] && ids[0] == ids[5]) // nested items belong to their top-level list238 #expect(result.document.list(ids[7])?.levels[0].start == 7)239240 // Bullets and numbers at the same level under one list need two definitions.241 let mixed = MarkdownImporter.importDocument("- a\n 1. x\n- b\n - y\n")242 #expect(Describe.blocks(mixed.document) == ["ListParagraph L0(•): a", "ListParagraph L1(1.): x", "ListParagraph L0(•): b", "ListParagraph L1(◦): y"])243 // Changing the bullet character starts a new list.244 let two = MarkdownImporter.importDocument("- a\n- b\n\n+ c\n")245 #expect(Set(two.document.body.compactMap { $0.paragraph?.list?.listID }).count == 2)246 // An empty item and an item that starts with a code block keep their bullet.247 #expect(blocks("-\n- x\n- code\n") == ["ListParagraph L0(•): ", "ListParagraph L0(•): x", "ListParagraph L0(•): ", "Code: code"])248 }249250 @Test func listLevelsAreClampedWithAWarning() {251 var md = ""252 for level in 0..<12 { md += String(repeating: " ", count: level) + "- level \(level)\n" }253 let result = MarkdownImporter.importDocument(md)254 let levels = result.document.body.compactMap { $0.paragraph?.list?.level }255 #expect(levels == [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 8, 8])256 #expect(Describe.texts(result.document).last == "level 11")257 #expect(result.warnings.contains { $0.feature == "Deep lists" && $0.severity == .approximated })258 }259260 @Test func codeBlocks() {261 #expect(blocks("````\n```\ninner fence\n```\n````\n") == ["Code: ```⏎inner fence⏎```"])262 #expect(blocks("~~~ python\n# not a heading\n*not em*\n~~~\nafter") == ["Code: # not a heading⏎*not em*", "Normal: after"])263 #expect(blocks("```\nnever closed\n\n- still code") == ["Code: never closed⏎⏎- still code"])264 #expect(blocks("text\n\n indented\n\n more\n\ntext") == ["Normal: text", "Code: indented⏎⏎more", "Normal: text"])265 #expect(blocks("```\n```\n") == ["Code: "])266 #expect(blocks("\tfoo\tbar\n") == ["Code: foo\tbar"])267 }268269 @Test func quotes() {270 let result = MarkdownImporter.importDocument("> first\n>\n> second\nlazy\n>\n> > nested\n\nout")271 #expect(Describe.blocks(result.document) == ["Quote: first", "Quote: second lazy", "Quote: nested", "Normal: out"])272 #expect(result.document.body[2].paragraph?.props.indentLeft == 56)273 #expect(result.warnings.contains { $0.feature == "Block quotes" })274 let mixed = MarkdownImporter.importDocument("> # Head\n> - item\n")275 #expect(Describe.blocks(mixed.document) == ["Heading1: Head", "ListParagraph L0(•): item"])276 #expect(mixed.warnings.contains { $0.feature == "Block quotes" && $0.detail.contains("Lists") })277 }278279 @Test func tables() throws {280 let result = MarkdownImporter.importDocument("""281 Intro282 | Name | Value | Note |283 |------|:-----:|------|284 | **a** | `1` |285 | b | 2 | 3 | 4 |286 c | d | e287288 a | b289 - | -290 1 | 2291 """)292 #expect(Describe.blocks(result.document) == [293 "Normal: Intro",294 "TABLE 3 cols",295 " H TableText: Name | TableText: Value | TableText: Note",296 " R TableText: {b:a} | TableText: {c:1} | TableText: ",297 " R TableText: b | TableText: 2 | TableText: 3",298 " R TableText: c | TableText: d | TableText: e",299 "TABLE 2 cols",300 " H TableText: a | TableText: b",301 " R TableText: 1 | TableText: 2",302 ])303 let table = try #require(result.document.body[1].table)304 #expect(table.columnWidths == [156, 156, 156])305 #expect(table.rows.allSatisfy { $0.cells.count == 3 })306 #expect(table.rows[1].cells[1].paragraphs[0].props.alignment == .center)307 #expect(result.warnings.contains { $0.feature == "Table cells" && $0.severity == .dropped })308 // Not tables: a delimiter with another column count, or no delimiter row at all.309 #expect(blocks("a | b\n---\n") == ["Heading2: a | b"])310 #expect(blocks("a | b\nc | d\n") == ["Normal: a | b c | d"])311 }312313 @Test func pandocSimpleTables() throws {314 let result = MarkdownImporter.importDocument("""315 Intro paragraph.316317 Right Left Center Default318 ------- ------ ---------- -------319 12 12 12 12320 123 123 123 123321322 Table: not a caption we interpret323324 - - -325326 -- --327 """)328 let lines = Describe.blocks(result.document)329 #expect(Array(lines[0...4]) == ["Normal: Intro paragraph.", "TABLE 4 cols",330 " H TableText: Right | TableText: Left | TableText: Center | TableText: Default",331 " R TableText: 12 | TableText: 12 | TableText: 12 | TableText: 12",332 " R TableText: 123 | TableText: 123 | TableText: 123 | TableText: 123"])333 #expect(lines[5] == "Normal: Table: not a caption we interpret")334 #expect(lines[6] == "RULE" && lines[7] == "RULE") // rules made of dashes and spaces are still rules335 let table = try #require(result.document.body[1].table)336 #expect(table.rows[1].cells.map { $0.paragraphs[0].props.alignment } == [.right, .left, .center, nil])337 }338339 @Test func footnotes() {340 let result = MarkdownImporter.importDocument("""341 One[^a] two[^b] again[^a] none[^zz].342343 [^b]: Second, with **bold** and a list:344345 - item346 [^a]: First.347 [^unused]: Never referenced.348 """)349 let doc = result.document350 let refs = doc.body[0].paragraph!.runs.compactMap { r -> NoteRef? in if case .note(let n)? = r.inline { return n } else { return nil } }351 #expect(refs.count == 3)352 #expect(refs[0].id == refs[2].id && refs[0].id != refs[1].id)353 #expect(doc.notes.count == 2)354 #expect(doc.notes[refs[0].id]?.paragraphs.map(\.visibleText) == ["First."])355 #expect(doc.notes[refs[1].id]?.paragraphs.map(\.visibleText) == ["Second, with bold and a list:", "item"])356 #expect(doc.notes[refs[1].id]?.paragraphs[0].style == StyleID.footnoteText)357 #expect(doc.notes[refs[1].id]?.paragraphs[1].list != nil)358 #expect(doc.body[0].paragraph?.visibleText == "One two again none[^zz].")359 #expect(Numbering(doc).noteNumbers[refs[0].id] == 1)360 #expect(result.warnings.contains { $0.feature == "Footnotes" && $0.severity == .dropped && $0.detail.contains("1 footnote definition") })361 }362363 @Test func mathBlocks() {364 #expect(blocks("$$ a + b $$\n\n$$\n\\begin{aligned}\na &= b \\\\\nc &= d\n\\end{aligned}\n$$\ntext") == [365 "Normal: <EQ a + b>", "Normal: <EQ \\begin{aligned}\na &= b \\\\\nc &= d\n\\end{aligned}>", "Normal: text"])366 #expect(blocks("$$ unclosed\n\nnext") == ["Normal: $$ unclosed", "Normal: next"])367 #expect(blocks("Price: $$\n") == ["Normal: Price: $$"])368 }369370 @Test func citations() throws {371 let doc = MarkdownImporter.importDocument("As shown [see @smith_2020, pp. 12--14; also -@doe.2019, chap. 3 and more; @x:y/z] and me@example.org or [@] or [not @ cite].").document372 let citation = try #require(doc.body[0].paragraph?.runs.compactMap { r -> Citation? in if case .citation(let c)? = r.inline { return c } else { return nil } }.first)373 #expect(citation.items == [374 CitationItem(key: "smith_2020", locator: "pp. 12--14", prefix: "see"),375 CitationItem(key: "doe.2019", locator: "chap. 3 and more", prefix: "also", suppressAuthor: true),376 CitationItem(key: "x:y/z"),377 ])378 #expect(doc.body[0].paragraph?.visibleText == "As shown and me@example.org or [@] or [not @ cite].")379 #expect(Numbering(doc).citedKeys == ["smith_2020", "doe.2019", "x:y/z"])380 // Suffix (no locator term) versus locator.381 #expect(CitationSyntax.parse("@k, emphasis added")?.items == [CitationItem(key: "k", suffix: "emphasis added")])382 #expect(CitationSyntax.parse("@k, 42")?.items == [CitationItem(key: "k", locator: "42")])383 #expect(CitationSyntax.parse("no key here") == nil)384 #expect(CitationSyntax.parse("@a; nothing") == nil)385 let item = CitationItem(key: "k", locator: "p. 1", prefix: "see", suffix: "etc.", suppressAuthor: true)386 #expect(CitationSyntax.render(Citation(items: [item, CitationItem(key: "z")])) == "see -@k, p. 1 etc.; @z")387 }388389 @Test func imagesComeOnlyFromDataURIsAndTheLoader() throws {390 let png = Fixture.png(width: 1000, height: 500), jpeg = Fixture.jpeg()391 var requested: [String] = []392 let md = """393   394   395    396 )) %22%2F%3E) 397 """398 let result = MarkdownImporter.importDocument(md) { path in399 requested.append(path)400 return path == "figures/a b.png" ? png : nil401 }402 // Remote, absolute and parent-directory sources never reach the loader.403 #expect(requested == ["figures/a b.png", "figures/a b.png", "nope.png"])404 let doc = result.document405 let images = Describe.texts(doc).joined()406 _ = images407 var objects: [ImageObject] = []408 DocumentModel.forEachParagraph(in: doc.body) { p, _ in for r in p.runs { if case .image(let i)? = r.inline { objects.append(i) } } }409 #expect(objects.map(\.altText) == ["local", "again", "jpeg"])410 #expect(objects[0].asset == objects[1].asset) // identical bytes are stored once411 #expect(doc.assets.count == 2)412 #expect(doc.assets[objects[2].asset]?.mimeType == "image/jpeg")413 // 1000 px at 96 dpi is 750 pt: scaled down to the text width, aspect ratio kept.414 #expect(objects[0].width == 468 && objects[0].height == 234)415 #expect(doc.body[0].paragraph?.visibleText.contains("missing") == true)416 #expect(Describe.texts(doc)[0].contains("remote proto ftp"))417 #expect(Describe.texts(doc)[0].contains("up abs file win"))418 #expect(Describe.texts(doc)[0].contains("svg bad"))419 let features = Dictionary(uniqueKeysWithValues: result.warnings.map { ($0.detail, $0.count) })420 #expect(features["Remote images are never downloaded; the alternative text was kept."] == 3)421 #expect(features["An image path outside the document folder was refused; the alternative text was kept."] == 4)422 #expect(features["A linked image file was not found; the alternative text was kept."] == 1)423 #expect(features["An image in an unsupported format (for example SVG) was skipped; the alternative text was kept."] == 1)424 #expect(features["An embedded image (data URI) could not be decoded; its alternative text was kept."] == 1)425 // Without a loader relative images are reported, not silently dropped.426 let none = MarkdownImporter.importDocument("")427 #expect(none.document.body[0].paragraph?.visibleText == "x")428 #expect(none.warnings.first?.detail == "Linked image files were not available; the alternative text was kept.")429 }430431 @Test func rawHTMLAndDivs() {432 let result = MarkdownImporter.importDocument("""433 <!-- a comment -->434 Text <!-- inline --> more435436 ::: {.note}437 Inside a div.438 :::439440 ::: {#refs}441 :::442 """)443 #expect(Describe.blocks(result.document) == ["Normal: Text more", "Normal: Inside a div.", "BIBLIOGRAPHY"])444 #expect(result.warnings.map(\.feature).sorted() == ["Fenced divs", "HTML comments"])445 }446447 @Test func lineEndingsAndControlCharacters() {448 #expect(blocks("# A\r\n\r\npara\rnext\r\n\r\n- x\r\n") == ["Heading1: A", "Normal: para next", "ListParagraph L0(•): x"])449 let result = MarkdownImporter.importDocument("nul\u{0}here \u{FFFC} bell\u{7}")450 #expect(result.document.body[0].paragraph?.visibleText == "nulhere \u{FFFD} bell")451 #expect(result.document.body[0].paragraph?.runs.allSatisfy { $0.inline == nil } == true)452 #expect(result.warnings.contains { $0.feature == "Control characters" })453 #expect(blocks("") == ["Normal: "])454 #expect(blocks("\n\n \n") == ["Normal: "])455 }456457 @Test func oversizedInputIsRefused() {458 // A string just above the limit, built cheaply.459 let big = String(repeating: "a", count: FormatLimits.maxTextBytes + 1)460 #expect(throws: FormatError.self) { _ = try MarkdownImporter.read(big) }461 let result = MarkdownImporter.importDocument(big)462 #expect(result.warnings.first?.severity == .dropped)463 #expect(result.warnings.first?.feature == "Unreadable Markdown")464 #expect(Describe.blocks(result.document) == ["Normal: "])465 }466}467468@Suite struct MarkdownExportTests {469 private func md(_ doc: DocumentModel, _ options: ExportOptions = ExportOptions()) -> String {470 String(decoding: MarkdownExporter.export(doc, options: options).data, as: UTF8.self)471 }472 private func doc(_ paragraphs: [Paragraph]) -> DocumentModel { DocumentModel(body: paragraphs.map(Block.paragraph)) }473474 @Test func specialCharactersSurviveARoundTrip() {475 let nasty = [476 "*x* _y_ **z** # not heading [link](u)  <b>tag</b> `tick` a|b ~t~ ^u^ back\\slash & & A @user me@x.org $5 and $x$ {curly} ~~no~~",477 "# heading?", "## two", "- dash", "+ plus", "* star", "1. one", "12) twelve", "> quote", "===", "---", "***", "___", ": colon", "% percent",478 "[^1]: not a note", "[ref]: http://not.a/definition", "| a | b |", "```", "~~~", "$$", "::: div", "<!-- comment -->", "http://bare.example/x and www.example.org", "<http://auto.example>",479 "ends with backslash\\", "tab\tinside", "trailing star*", "*", "_", "a * b", "1986. A great year", "#hashtag", "+1", "-1", "=",480 "line\u{2028}# not heading\u{2028}- not list\u{2028}1. not ordered",481 "Ünïcödé 中文 العربية עברית 👩👩👧👦 e\u{301} 🇨🇦",482 ]483 let original = doc(nasty.map { Paragraph($0) })484 let text = md(original)485 let back = MarkdownImporter.importDocument(text)486 #expect(Describe.texts(back.document) == nasty, "\(text)")487 #expect(back.document.body.allSatisfy { $0.paragraph?.style == StyleID.normal && $0.paragraph?.list == nil })488 // No run picked up formatting, a link, or an inline object along the way.489 for (k, block) in back.document.body.enumerated() {490 #expect(block.paragraph?.runs.allSatisfy { $0.props == .none && $0.inline == nil && $0.charStyle == nil } == true, "paragraph \(k): \(nasty[k])")491 }492 #expect(back.warnings.isEmpty)493 }494495 @Test func emphasisMarkersAlwaysTouchWords() {496 let b = CharProps(bold: true), i = CharProps(italic: true), bi = CharProps(bold: true, italic: true)497 let cases: [([Run], String)] = [498 ([Run("bold ", props: b), Run("plain")], "**bold** plain"),499 ([Run("a"), Run(" bold", props: b)], "a **bold**"),500 ([Run("two words", props: b), Run(" and "), Run("it", props: i)], "**two words** and *it*"),501 ([Run("bold", props: b), Run("italic", props: i)], "**bold**<!---->*italic*"),502 ([Run("a", props: b), Run("b", props: bi), Run("c", props: i)], "**a*b***<!---->*c*"),503 ([Run("Note:", props: b), Run("text")], "**Note:**<!---->text"),504 ([Run("pre"), Run("(bold)", props: b)], "pre<!---->**(bold)**"),505 ([Run("in"), Run("tra", props: i), Run("word")], "in*tra*word"),506 ([Run(" spaced ", props: i)], "*spaced*"),507 ([Run("x", props: CharProps(strikethrough: true)), Run("y", props: CharProps(underline: true)), Run("z", props: CharProps(highlight: .white))], "~~x~~<u>y</u><mark>z</mark>"),508 ([Run("H"), Run("2", props: CharProps(vertical: .subscript)), Run("O"), Run("n", props: CharProps(vertical: .superscript))], "H<sub>2</sub>O<sup>n</sup>"),509 ([Run("a `b` c", charStyle: StyleID.inlineCode), Run(" "), Run("`", charStyle: StyleID.inlineCode), Run("x", props: b, charStyle: StyleID.inlineCode)], "``a `b` c`` `` ` ``**`x`**"),510 ([Run("tick`"), Run("code", charStyle: StyleID.inlineCode), Run("more", props: i, charStyle: StyleID.inlineCode)], "tick\\`<!---->`code`*`more`*"),511 ([Run("strong style", charStyle: StyleID.strong), Run(" "), Run("emph", charStyle: StyleID.emphasis)], "**strong style** *emph*"),512 ([Run("go to ", props: CharProps(link: "https://e.org/a b(c)")), Run("site", props: CharProps(bold: true, link: "https://e.org/a b(c)"))], "[go to **site**](<https://e.org/a b(c)>)"),513 ]514 for (runs, expected) in cases {515 let original = doc([Paragraph(runs: runs)])516 let text = md(original).trimmingCharacters(in: .newlines)517 #expect(text == expected)518 // And every one of them reads back with the same formatting on the same characters.519 let back = MarkdownImporter.importDocument(text).document520 var want = original.body[0].paragraph!521 for k in want.runs.indices { want.runs[k].text = want.runs[k].text }522 let lhs = Describe.paragraph(back.body[0].paragraph!, back).replacingOccurrences(of: " ", with: "")523 var trimmed = want524 trimmed.runs = want.runs525 let rhs = Describe.paragraph(trimmed, original).replacingOccurrences(of: " ", with: "")526 #expect(lhs == rhs, "\(expected)")527 }528 }529530 @Test func structureIsWritten() {531 var document = DocumentModel(meta: DocumentMetadata(title: "T \"q\"", author: "A", subject: "S", keywords: ["k1", "k 2"], language: "de"))532 document.lists = [ListDefinition.bullets(id: "b"), ListDefinition.numbered(id: "n"), ListDefinition.numbered(id: "n2"), ListDefinition.bullets(id: "b2")]533 document.lists[1].levels[0].start = 9534 func item(_ text: String, _ list: String, _ level: Int = 0) -> Block { .paragraph(Paragraph(style: StyleID.listParagraph, runs: [Run(text)], list: ListItem(listID: list, level: level))) }535 document.body = [536 .paragraph(Paragraph("Title", style: StyleID.title)),537 .paragraph(Paragraph(style: StyleID.heading(2), runs: [Run(inline: .bookmark("sec-a")), Run("Section {a}\u{2028}two")])),538 .paragraph(Paragraph("Deep", style: StyleID.heading(8))),539 item("nine", "n"), item("ten", "n"), item("sub", "n", 1), item("subsub", "n", 2), item("eleven", "n"),540 .paragraph(Paragraph(style: StyleID.listParagraph, props: ParaProps(indentLeft: 36), runs: [Run("continued")])),541 item("first of another numbered list", "n2"),542 item("☐ todo", "b"), item("☑ done", "b"), item("jumped level", "b", 3),543 item("another bullet list", "b2"),544 .paragraph(Paragraph("q1", style: StyleID.quote)), .paragraph(Paragraph("q2\u{2028}q2b", style: StyleID.quote)),545 .paragraph(Paragraph("code ``` here", style: StyleID.code)), .paragraph(Paragraph("second code paragraph", style: StyleID.code)),546 .paragraph(ModelIdiom.rule()),547 .paragraph(Paragraph()),548 .pageBreak(BreakBlock()),549 .paragraph(Paragraph(runs: [Run(inline: .equation(Equation(latex: "x", display: true)))])),550 .paragraph(Paragraph("Sub", style: StyleID.subtitle)),551 ]552 let result = MarkdownExporter.export(document)553 #expect(String(decoding: result.data, as: UTF8.self) == """554 ---555 title: "T \\"q\\""556 author: "A"557 subject: "S"558 keywords: ["k1", "k 2"]559 lang: "de"560 ---561562 # Title563564 ## Section \\{a\\} two {#sec-a}565566 ###### Deep567568 9. nine569 10. ten570 1. sub571 1. subsub572 11. eleven573574 continued575576 1) first of another numbered list577578 - [ ] todo579 - [x] done580 - jumped level581582 * another bullet list583584 > q1585 >586 > q2\\587 > q2b588589 ````590 code ``` here591 second code paragraph592 ````593594 ---595596 $$597 x598 $$599600 Sub601602 """)603 let features = result.warnings.map(\.feature)604 #expect(features.contains("Headings") && features.contains("Title") && features.contains("Page breaks") && features.contains("Page setup"))605 // Reading it back gives the same structure (title and deep heading are approximated as documented).606 let back = MarkdownImporter.importDocument(String(decoding: result.data, as: UTF8.self))607 #expect(Describe.blocks(back.document) == [608 "Heading1: Title", "Heading2: <bm sec-a>Section {a} two", "Heading6: Deep",609 "ListParagraph L0(9.): nine", "ListParagraph L0(10.): ten", "ListParagraph L1(1.): sub", "ListParagraph L2(1.): subsub", "ListParagraph L0(11.): eleven",610 "ListParagraph: continued", "ListParagraph L0(1.): first of another numbered list",611 "ListParagraph L0(•): ☐ todo", "ListParagraph L0(•): ☑ done", "ListParagraph L1(◦): jumped level", "ListParagraph L0(•): another bullet list",612 "Quote: q1", "Quote: q2⏎q2b", "Code: code ``` here⏎second code paragraph", "RULE", "Normal: <EQ x>", "Normal: Sub",613 ])614 #expect(back.document.meta.title == "T \"q\"" && back.document.meta.keywords == ["k1", "k 2"] && back.document.meta.language == "de")615 #expect(Set(back.document.body.compactMap { $0.paragraph?.list?.listID }).count == 4)616 #expect(back.warnings.isEmpty)617 }618619 @Test func tablesAreWrittenAsPipeTables() {620 var table = Table(strings: [["Name", "A|B"], ["x\u{2028}y", "*z*"]], width: 300)621 table.rows[0].cells[1].paragraphs[0].props.alignment = .right622 table.rows[1].cells[0].paragraphs.append(Paragraph("second", style: StyleID.tableText))623 var merged = Table(columnWidths: [50, 50, 50], rows: [624 TableRow(cells: [TableCell(paragraphs: [Paragraph("wide")], colSpan: 2), TableCell(paragraphs: [Paragraph("tall")], vMerge: .restart)]),625 TableRow(cells: [TableCell("a"), TableCell("b"), TableCell(vMerge: .continue)]),626 ])627 merged.rows[0].isHeader = false628 let result = MarkdownExporter.export(DocumentModel(body: [.table(table), .table(merged)]))629 #expect(String(decoding: result.data, as: UTF8.self) == """630 | **Name** | **A\\|B** |631 | --- | --: |632 | x<br>y<br>second | \\*z\\* |633634 | wide | | tall |635 | --- | --- | --- |636 | a | b | |637638 """)639 #expect(result.warnings.contains { $0.feature == "Merged table cells" && $0.severity == .approximated })640 #expect(result.warnings.contains { $0.feature == "Tables" && $0.detail.contains("header row") })641 let back = MarkdownImporter.importDocument(String(decoding: result.data, as: UTF8.self)).document642 #expect(Describe.blocks(back)[0...2] == ["TABLE 2 cols", " H TableText: {b:Name} | TableText: {b:A|B}", " R TableText: x⏎y⏎second | TableText: *z*"])643 }644645 @Test func richDocumentExport() throws {646 let source = Fixture.rich()647 let result = MarkdownExporter.export(source)648 let text = String(decoding: result.data, as: UTF8.self)649 // Captions, sequences and cross-references are written as their resolved text.650 #expect(text.contains("\nFigure 1: A caption\n"))651 #expect(text.contains("Figure 1[?](#intro)")) // "Figure 1", then the page reference: unknown here, still a link to its target652 #expect(text.contains("[see @knuth1984, p. 4; -@doe:2020, passim]"))653 #expect(text.contains("$e^{i\\pi}+1=0$"))654 #expect(text.contains("$$\n\\int_0^1 x^2\\,dx = \\frac{1}{3}\n$$"))655 #expect(text.contains("[^1]") && text.contains("[^en1]"))656 #expect(text.contains("[^1]: A footnote with *emphasis*."))657 #expect(text.contains("[^en1]: An endnote.\n\n Second paragraph."))658 #expect(text.contains(")659 #expect(text.contains("**Sommaire**\n\n- [Introduction](#intro)\n - [Second section](#second-section)"))660 #expect(text.contains("# Références\n\n::: {#refs}\n:::"))661 #expect(text.contains(" inserted") && !text.contains("deleted")) // changes accepted by default662 #expect(text.contains("Rich fixture2026") == false)663 // Everything Markdown cannot carry is reported, by name.664 let features = Set(result.warnings.map(\.feature))665 for expected in ["Tracked changes", "Comments", "Headers and footers", "Sections", "Columns", "Page decoration", "Shapes", "Page-dependent fields", "Fields",666 "Cross-references", "Preserved DOCX content", "Citations", "Bibliography", "Page setup", "Fonts", "Colours", "Highlight", "Underline",667 "Character effects", "Paragraph formatting", "Custom styles", "Page breaks", "Floating objects", "Merged table cells", "Table formatting",668 "Endnotes", "Generated lists", "Title", "Image sizes"] {669 #expect(features.contains(expected), "missing warning: \(expected)")670 }671 #expect(result.warnings.first { $0.feature == "Comments" }?.severity == .dropped)672 #expect(result.warnings.first { $0.feature == "Tracked changes" }?.detail.contains("accepted") == true)673674 // Import it back: text, structure and semantic objects that Markdown can express are all there.675 let back = MarkdownImporter.importDocument(text).document676 let lines = Describe.blocks(back)677 #expect(lines[0] == "Heading1: Rich fixture")678 #expect(lines.contains("Heading1: <bm intro>Introduction"))679 #expect(lines[1...3] == ["Normal: {b:Sommaire}", "ListParagraph L0(•): [Introduction](#intro)", "ListParagraph L1(◦): [Second section](#second-section)"])680 #expect(lines.contains("Heading2: <bm second-section>Second section")) // the heading a link points at gets its anchor back681 #expect(lines.contains("Normal: <EQ \\int_0^1 x^2\\,dx = \\frac{1}{3}>"))682 #expect(lines.contains("Normal: Figure 1: A caption"))683 #expect(lines.contains("ListParagraph L0(•): First bullet") && lines.contains("ListParagraph L1(◦): Nested bullet"))684 #expect(lines.contains("ListParagraph L0(1.): One") && lines.contains("ListParagraph L1(1.): One point one"))685 #expect(lines.contains("Quote: A quotation.") && lines.contains("Code: let x = 1⏎print(x)") && lines.contains("BIBLIOGRAPHY"))686 #expect(lines.contains(" H TableText: Header A+B | TableText: | TableText: Tall"))687 #expect(lines.contains(" R TableText: a2 | TableText: line one⏎line two | TableText: 42"))688 let mixed = try #require(lines.first { $0.contains("Plain") })689 #expect(mixed.contains("{b:bold} {i:italic} [link](https://example.org/a?b=1&c=2)"))690 #expect(mixed.contains("<img 30x23 alt=A blue box data=\(source.assets["img/1 é"]!.data.count)>"))691 #expect(mixed.contains("<eq e^{i\\pi}+1=0>"))692 #expect(mixed.contains("<footnote FootnoteText: A footnote with {i:emphasis}.>"))693 #expect(mixed.contains("<footnote FootnoteText: An endnote. / FootnoteText: Second paragraph.>"))694 #expect(mixed.contains("<cite see|knuth1984|p. 4|;|-doe:2020||passim>"))695 #expect(mixed.contains("end⏎second line\ttabbed"))696 #expect(back.assets.count == 2)697 }698699 @Test func exportOptions() {700 let source = Fixture.rich()701 var options = ExportOptions()702 #expect(options.acceptChanges && options.citations == nil && options.mathML == nil)703 options.acceptChanges = false704 let rejected = MarkdownExporter.export(source, options: options)705 let rejectedText = String(decoding: rejected.data, as: UTF8.self)706 #expect(rejectedText.contains(" deleted") && !rejectedText.contains("inserted"))707 #expect(rejectedText.contains("**Plain bold**")) // the tracked format change is undone: "Plain " was bold before708 #expect(rejected.warnings.first { $0.feature == "Tracked changes" }?.detail.contains("rejected") == true)709710 options = ExportOptions()711 options.citations = StubCitations()712 options.imagePath = { asset in "media/\(asset.id.count).\(asset.fileExtension)" }713 let formatted = MarkdownExporter.export(source, options: options)714 let text = String(decoding: formatted.data, as: UTF8.self)715 #expect(text.contains("(KNUTH1984 p. 4; DOE:2020)"))716 #expect(!text.contains("[@knuth1984"))717 #expect(text.contains("# Références\n\n\\[1\\] *The TeXbook*\n\n\\[2\\] *On Things*"))718 #expect(text.contains("") && text.contains(""))719 #expect(!text.contains("base64"))720 #expect(formatted.warnings.first { $0.feature == "Citations" }?.detail.contains("formatted text") == true)721 #expect(!formatted.warnings.contains { $0.feature == "Bibliography" })722 }723724 @Test func paragraphMarkRevisionsJoinOrSplit() {725 var first = Paragraph(id: "a", runs: [Run("Hello ")])726 first.markRevision = Revision(kind: .deletion, author: "x")727 var second = Paragraph(id: "b", runs: [Run("world")])728 second.markRevision = Revision(kind: .insertion, author: "x")729 let third = Paragraph(id: "c", runs: [Run("!")])730 let document = DocumentModel(body: [.paragraph(first), .paragraph(second), .paragraph(third)])731 #expect(md(document) == "Hello world\n\n!\n") // accepted: the deleted mark joins a+b, the inserted mark stays732 var options = ExportOptions()733 options.acceptChanges = false734 #expect(md(document, options) == "Hello\n\nworld!\n") // rejected: a keeps its mark, the inserted mark disappears735 }736737 @Test func exportIsStableAfterOneRoundTrip() {738 let first = md(Fixture.rich())739 let second = md(MarkdownImporter.importDocument(first).document)740 let third = md(MarkdownImporter.importDocument(second).document)741 #expect(second == third)742 #expect(Describe.blocks(MarkdownImporter.importDocument(second).document) == Describe.blocks(MarkdownImporter.importDocument(third).document))743 }744745 @Test func missingAssetIsReported() {746 let document = DocumentModel(body: [.paragraph(Paragraph(runs: [Run("See "), Run(inline: .image(ImageObject(asset: "gone", width: 10, height: 10, altText: "the chart")))]))])747 let result = MarkdownExporter.export(document)748 #expect(String(decoding: result.data, as: UTF8.self) == "See the chart\n")749 #expect(result.warnings.contains { $0.feature == "Missing images" && $0.severity == .dropped })750 }751}752