Repository Public
1// SP Notes Core2// File: Tests/SPNotesDocumentTests/TextTests.swift3// Description: Plain text, word count, preview, references, Markdown export and import, HTML export — checked4// against outputs of the web application's own helpers stored in the fixtures.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation9import Testing10@testable import SPNotesDocument1112@Suite struct TextUtilityTests {13 @Test(arguments: [("rich", 0), ("unknown", 1)])14 func matchesTheServer(_ name: String, _ index: Int) throws {15 let doc = try Fixture.doc(name)16 let expected = try Fixture.expected(index)17 #expect(doc.plainText() == expected["text"]?.stringValue)18 #expect(doc.wordCount == expected["words"]?.intValue)19 #expect(doc.markdown() == expected["markdown"]?.stringValue)20 #expect(doc.pageReferences() == expected["refs"]?.arrayValue?.compactMap(\.stringValue))21 }2223 @Test func plainTextRules() throws {24 let text = try Fixture.doc("rich").plainText()25 #expect(text.hasPrefix("Carnet de recherche — séance 9\nTexte gras, italique, souligné, barré, code(), surligné, H2O, E=mc2 et gras italique lié.\nVoir @Plan de cours et la page liée, puis e^{i\\pi}+1=0 avec une note.\nDeuxième ligne après un saut.\n\nListes imbriquées\nPremier point\nSous-point A"))26 #expect(text.contains("let x = 1\nprint(x)") && text.contains("PosteMontant ($)Note\nLoyer | brut1 200ligne 1\nligne 2\nFichier") && text.hasSuffix("Centré\n\nSource : registre foncier."))27 #expect(!text.contains("\n\n\n"))28 #expect(NoteDocument().plainText() == "" && NoteDocument().wordCount == 0 && NoteDocument().markdown() == "\n")29 let odd = try NoteDocument(json: Data(#"{"type":"doc","content":[{"type":"paragraph","content":[{"type":"mention","attrs":{"id":"x"}},{"type":"mention","attrs":{"label":12}},{"type":"inlineMath"},{"type":"hardBreak"},{"type":"text","text":" fin "}]}]}"#.utf8))30 #expect(odd.plainText() == "@@12\n\u{a0} fin")31 }3233 @Test(arguments: [("", 0), ("un deux trois", 3), ("l'été can’t re-do 3x", 4), ("a--b c'- 'd'", 4), ("naïve café № 5 ٣ 日本語", 5), ("x_y z.w a@b", 6),34 ("don't-stop'n'go", 1), ("😀 — … !", 0), ("e\u{301}t", 2)])35 func wordCount(_ text: String, _ count: Int) {36 #expect(NoteDocument.wordCount(of: text) == count)37 }3839 @Test func preview() throws {40 let doc = try Fixture.doc("rich")41 #expect(doc.preview(maxLength: 200) == "Carnet de recherche — séance 9")42 #expect(doc.preview(maxLength: 10) == "Carnet de…")43 #expect(doc.preview(maxLength: 7) == "Carnet…" && doc.preview(maxLength: 1) == "…" && doc.preview(maxLength: 0) == "")44 #expect(md("\n\n \n deux mots\tici \nsuite").preview(maxLength: 50) == "deux mots ici")45 #expect(NoteDocument().preview(maxLength: 50) == "")46 #expect(md("```\n\n code\n```").preview(maxLength: 50) == "code")47 #expect(md("👨👩👧👦👨👩👧👦👨👩👧👦").preview(maxLength: 2) == "👨👩👧👦…")48 }4950 @Test func references() throws {51 let doc = try Fixture.doc("rich")52 #expect(doc.pageReferences() == ["3f2b8c1e-7a4d-4e9b-9c1a-5d6e7f8a9b0c", "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"])53 #expect(doc.attachmentReferences() == ["aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee", "9d8c7b6a-5e4f-4d3c-8b2a-1f0e9d8c7b6a", "11111111-2222-4333-8444-555555555555"])54 #expect(doc.taskItems() == TaskCounts(total: 5, checked: 2))55 let json = #"""56 {"type":"doc","content":[{"type":"paragraph","content":[57 {"type":"mention","attrs":{"id":"u1","kind":"user"}},{"type":"mention","attrs":{"id":"p1","kind":"page"}},{"type":"mention","attrs":{"id":""}},{"type":"mention","attrs":{"id":"p1"}},58 {"type":"text","text":"a","marks":[{"type":"link","attrs":{"href":"/p/0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d#h"}}]},59 {"type":"text","text":"b","marks":[{"type":"link","attrs":{"href":"https://x.org/p/0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4e"}}]},60 {"type":"text","text":"c","marks":[{"type":"link","attrs":{"href":"/p/0A1B2C3D-4e5f-4a6b-8c7d-9e0f1a2b3c4d"}}]},61 {"type":"text","text":"d","marks":[{"type":"link","attrs":{"href":"/p/short"}},{"type":"link","attrs":{"href":7}}]}]},62 {"type":"image","attrs":{"src":"https://example.org/api/files/11111111-2222-4333-8444-555555555555"}},63 {"type":"image","attrs":{"src":"/api/files/22222222-2222-4333-8444-555555555555?w=200"}},64 {"type":"fileBlock","attrs":{"src":"/api/files/nope","attachmentId":"33333333-2222-4333-8444-555555555555"}},65 {"type":"fileBlock","attrs":{"src":"/api/files/22222222-2222-4333-8444-555555555555","attachmentId":null}},66 {"type":"youtube","attrs":{"src":"/api/files/44444444-2222-4333-8444-555555555555"}}]}67 """#68 let odd = try NoteDocument(json: Data(json.utf8))69 #expect(odd.pageReferences() == ["p1", "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"])70 #expect(odd.attachmentReferences() == ["22222222-2222-4333-8444-555555555555", "33333333-2222-4333-8444-555555555555"])71 #expect(odd.taskItems() == TaskCounts(total: 0, checked: 0))72 }73}7475@Suite struct MarkdownExportTests {76 @Test func exportDetails() throws {77 let mdText = try Fixture.doc("rich").markdown()78 #expect(mdText.hasPrefix("# Carnet de recherche — séance 9\n\nTexte **gras**, *italique*, <u>souligné</u>, ~~barré~~, `code()`, ==surligné==, H<sub>2</sub>O, E=mc<sup>2</sup> et ***[gras italique lié](https://example.org/a?b=1&c=2)***.\n"))79 #expect(mdText.contains("- Premier point\n - Sous-point A\n - Sous-point B\n 1. Étape une\n 2. Étape deux\n- Deuxième point\nParagraphe de suite dans le même item.\n- \n1. Un\n2. Deux\n```swift\nlet x = 1\nprint(x)\n```\n"))80 #expect(mdText.contains("| Loyer \\| brut | **1 200** | ligne 1ligne 2 |\n| Fichier | |\n"))81 #expect(mdText.contains("> [!WARNING] ⚠️\n> Attention aux points suivants :\n> - premier\n> - second\n"))82 #expect(mdText.contains("<details><summary>Détails **repliables**</summary>\n\nContenu caché.\n\n- [ ] dans les détails\n\n</details>\n"))83 #expect(mdText.hasSuffix("[^1]: Source : [registre foncier](https://example.org/registre) .\n"))84 }8586 @Test func javaScriptStringSemantics() throws {87 let json = #"""88 {"type":"doc","content":[89 {"type":"heading","content":[{"type":"text","text":"sans niveau"}]},{"type":"heading","attrs":{"level":2.0},"content":[{"type":"text","text":"deux"}]},90 {"type":"image"},{"type":"image","attrs":{"src":null,"alt":null}},{"type":"fileBlock","attrs":{"name":"n"}},{"type":"youtube"},91 {"type":"codeBlock","attrs":{"language":null},"content":[{"type":"text","text":"x"}]},92 {"type":"paragraph","content":[{"type":"text","text":"l","marks":[{"type":"link"}]},{"type":"mention","attrs":{"id":"i"}},{"type":"mention"},{"type":"footnoteRef"},{"type":"inlineMath"}]},93 {"type":"callout","content":[{"type":"paragraph","content":[{"type":"text","text":"c"}]}]},94 {"type":"aiBlock","content":[{"type":"paragraph"}]},{"type":"details"},{"type":"table"},{"type":"table","content":[{"type":"tableRow"}]},95 {"type":"taskList","content":[{"type":"taskItem","attrs":{"checked":1},"content":[{"type":"paragraph","content":[{"type":"text","text":" t"}]}]},{"type":"taskItem"}]},96 {"type":"blockMath"},{"type":"footnote"},{"type":"widget","content":[{"type":"paragraph","content":[{"type":"text","text":"w"}]}]}97 ]}98 """#99 let out = try NoteDocument(json: Data(json.utf8)).markdown()100 // Expected values produced by the web application's toMarkdown / docText / wordCount on this very document.101 #expect(out == "# sans niveau\n\n## deux\n\n\n\n\n\n[📎 n](undefined)\n\nundefined\n\n```\nx\n```\n\n[l](undefined)[[i]][[undefined]][^undefined]$$\n\n> [!NOTE] \n> c\n\n> [!AI] \n\n<details><summary></summary>\n\n</details>\n\n| |\n| |\n\n- [x] t\n- [ ] \n$$\n\n$$\n\n[^undefined]: \n\nw\n")102 let doc = try NoteDocument(json: Data(json.utf8))103 #expect(doc.plainText() == "sans niveau\ndeux\n\nx\nl@@\nc\n\n t\n\nw" && doc.wordCount == 8 && doc.pageReferences() == ["i"])104 }105}106107@Suite struct ServerParityTests {108 /// 400 arbitrary trees (unknown types, odd attributes, stray text) against outputs recorded from the web109 /// application's docText / wordCount / toMarkdown. A null entry means the server helper threw on that tree.110 @Test func randomTreesMatchTheServer() throws {111 let server = try #require(try JSONValue(data: Fixture.data("random-server")).arrayValue)112 var r = SplitMix64(seed: 1234)113 var compared = 0114 for (i, expected) in server.enumerated() {115 let doc = NoteDocument(node: Random.anyNode(&r, depth: 1 + r.int(5), type: "doc"))116 if let text = expected["text"]?.stringValue { #expect(doc.plainText() == text, "tree \(i)"); compared += 1 }117 if let words = expected["words"]?.intValue { #expect(doc.wordCount == words, "tree \(i)") }118 if let markdown = expected["markdown"]?.stringValue { #expect(doc.markdown() == markdown, "tree \(i)"); compared += 1 }119 }120 #expect(server.count == 400 && compared >= 790)121 }122}123124@Suite struct MarkdownImportTests {125 /// Samples whose server output needs the fixes the full server import applies afterwards, or that the server126 /// leaves invalid for the schema; they have explicit expectations below.127 static let special: Set<Int> = [7, 20, 21, 29, 38, 41]128129 /// The server's raw import made comparable: marks made valid, adjacent text merged.130 func normalized(_ n: PMNode) -> PMNode {131 var n = n132 if n.type == "text" { n.marks = n.marks.map(InlineTools.validMarks) }133 guard let kids = n.content else { return n }134 var out: [PMNode] = []135 for k in kids.map(normalized) {136 if k.type == "text", let last = out.last, last.type == "text", last.marks == k.marks { out[out.count - 1].text! += k.text ?? "" } else { out.append(k) }137 }138 n.content = out139 return n140 }141142 @Test func matchesTheServerOnEverySample() throws {143 let samples = try #require(try JSONValue(data: Fixture.data("md-samples")).arrayValue).compactMap(\.stringValue)144 let server = try #require(try JSONValue(data: Fixture.data("md-server")).arrayValue)145 #expect(samples.count == 48 && server.count == 48)146 for (i, sample) in samples.enumerated() {147 let doc = NoteDocument(markdown: sample)148 expectValid(doc, "sample \(i)")149 var ensured = doc150 ensured.ensureBlockIds()151 #expect(ensured == doc, "sample \(i): ids must already be there")152 guard !Self.special.contains(i) else { continue }153 let theirs = normalized(try PMNode(json: server[i]))154 #expect(sexp(doc) == sexp(theirs), "sample \(i): \(sample.debugDescription)")155 #expect(doc.plainText() == TextTools.docText(theirs), "sample \(i)")156 }157 }158159 @Test func structures() {160 #expect(sexp(md("- [ ] à faire\n- [x] fait\n - [ ] sous-tâche\n- simple")) == #"tl(ti[ ](p("à faire")) ti[x](p("fait") tl(ti[ ](p("sous-tâche")))) ti[ ](p("simple")))"#)161 #expect(sexp(md("1. un\n2. deux\n 1. sous\n3. trois")) == #"ol(li(p("un")) li(p("deux") ol(li(p("sous")))) li(p("trois")))"#)162 #expect(md("3. trois\n4. quatre").lines[0].path[0].attrs["start"] == 3)163 #expect(sexp(md("> a\n>\n> > b\n\n---\n\n```swift\nlet a = 1\n\nlet b = 2\n```")) == "bq(p(\"a\") bq(p(\"b\"))) hr code(\"let a = 1\n\nlet b = 2\")")164 #expect(sexp(md("| A | B |\n| --- | :-: |\n| 1 | **2** |")) == #"table(tableRow(tableHeader(p("A")) tableHeader(p("B"))) tableRow(tableCell(p("1")) tableCell(p("2"[bold]))))"#)165 #expect(sexp(md("$$\nx^2\n$$\n\nInline $a+b$ fin")) == #"blockMath p("Inline " inlineMath " fin")"#)166 #expect(md("$$\nx^2\n$$").lines[0].attrs["latex"] == "x^2")167 #expect(sexp(md("a \nb\\\nc\nd")) == "p(\"a\" br \"b\" br \"c\nd\")")168 #expect(sexp(md("#### Quatre")) == #"h3("Quatre")"#)169 #expect(sexp(md("")) == "p" && sexp(md(" \n\n")) == "p")170 let code = md("```js\nx\n```\n\n```\ny\n```")171 #expect(code.lines[0].attrs["language"] == "js" && code.lines[1].attrs["language"] == .some(nil))172 }173174 @Test func imagesBecomeBlocks() {175 let d = md("\n\ntexte  suite\n\n et \n\n# Titre  fin\n\n- ")176 #expect(sexp(d) == #"image p("texte ") image p(" suite") p("[image: rel] et [image: rel2.png]") h1("Titre ") image h1(" fin") ul(li(p image))"#)177 #expect(d.lines[0].attrs["src"] == "https://example.org/i.png" && d.lines[0].attrs["alt"] == "alt" && d.lines[0].attrs["title"] == .some(nil))178 #expect(d.attachmentReferences() == ["11111111-2222-4333-8444-555555555555"])179 expectValid(d)180 }181182 @Test func wikiLinks() {183 #expect(sexp(md("Un [[Lien Wiki]] et [[Cible|alias]] et [[Page#section]].")) == #"p("Un Lien Wiki et alias et Page.")"#)184 let d = NoteDocument(markdown: "Voir [[Plan de cours|le plan]] et [[Inconnue]].") { $0 == "Plan de cours" ? "3f2b8c1e-7a4d-4e9b-9c1a-5d6e7f8a9b0c" : nil }185 #expect(sexp(d) == #"p("Voir " mention " et Inconnue.")"#)186 #expect(d.pageReferences() == ["3f2b8c1e-7a4d-4e9b-9c1a-5d6e7f8a9b0c"] && d.plainText() == "Voir @le plan et Inconnue.")187 expectValid(d)188 }189190 @Test func alwaysValidForTheSchema() {191 // What the server leaves invalid is repaired: heading or block first in an item, empty quote, marks with code.192 #expect(sexp(md("- # titre\n- > q\n- ```\n c\n ```")) == #"ul(li(p h1("titre")) li(p bq(p("q"))) li(p code("c")))"#)193 #expect(sexp(md("a\n\n> \n\nc")) == #"p("a") bq(p) p("c")"#)194 #expect(sexp(md("**`a`** [`b`](https://example.org) ***c***")) == #"p("a"[code] " " "b"[code] " " "c"[bold,italic])"#)195 #expect(sexp(md("[x](javascript:alert(1))  [ok](https://example.org)")) == #"p("x [image: y] " "ok"[link])"#)196 var r = SplitMix64(seed: 5)197 let pieces = ["# ", "- ", " - ", "1. ", "> ", "```\n", "| a | b |\n| - | - |\n", "**", "*", "`", "~~", "[x](y)", "", "$a$", "$$", "\n", "\n\n", "mot ", "- [x] ",198 " ", "---\n", "<b>", "\\", "[[w]]", "_", "&", "https://e.org ", "===\n", " ", "é😀"]199 for i in 0..<1500 {200 let text = (0..<(1 + r.int(14))).map { _ in r.pick(pieces) }.joined()201 let d = NoteDocument(markdown: text)202 let issues = d.schemaViolations(strict: true)203 #expect(issues.isEmpty && !d.lines.isEmpty, "random markdown \(i) \(text.debugDescription): \(issues.prefix(3))")204 }205 }206207 @Test func exportThenImportKeepsTheText() throws {208 for source in ["# T\n\nUn **gras** et *ital* et `code`.\n\n- a\n - b\n- c\n\n> q\n\n```swift\nlet x = 1\n```\n\n| A | B |\n| --- | --- |\n| 1 | 2 |\n\n---\n\n$$\nx\n$$\n",209 "- [x] fait\n- [ ] non\n\n1. un\n2. deux\n"] {210 let d = md(source)211 #expect(d.markdown() == source)212 #expect(md(d.markdown()).isStructurallyEqual(to: d) == false, "ids differ")213 #expect(sexp(md(d.markdown())) == sexp(d))214 }215 }216}217218@Suite struct HTMLTests {219 @Test func standaloneDocument() throws {220 let html = try Fixture.doc("rich").html(title: "Séance <9> & \"co\"")221 #expect(html.hasPrefix("<!doctype html>\n<html lang=\"en\"><head><meta charset=\"utf-8\">"))222 #expect(html.contains("<title>Séance <9> & "co"</title>") && html.contains("<h1>Séance <9> & "co"</h1>"))223 #expect(html.contains("<style>") && !html.contains("<script") && !html.contains("<link ") && html.hasSuffix("</main></body></html>\n"))224 for piece in ["<h1>Carnet de recherche — séance 9</h1>", "<strong>gras</strong>", "<em>italique</em>", "<u>souligné</u>", "<s>barré</s>", "<code>code()</code>",225 "<mark>surligné</mark>", "H<sub>2</sub>O", "E=mc<sup>2</sup>", #"<em><strong><a href="https://example.org/a?b=1&c=2">gras italique lié</a></strong></em>"#,226 #"<span class="mention">@Plan de cours</span>"#, #"<code class="math">e^{i\pi}+1=0</code>"#, ##"<sup class="fn"><a href="#fn-1">1</a></sup>.<br>Deuxième"##,227 "<ul><li><p>Premier point</p>\n<ul><li><p>Sous-point A</p></li>", #"<ol start="3"><li><p>Étape une</p></li>"#,228 #"<ul class="tasks"><li class="done"><input type="checkbox" disabled checked> <div><p>Tâche faite</p></div></li><li class=""><input type="checkbox" disabled > <div><p>Tâche à faire</p>"#,229 "<blockquote><p>Une citation sur deux paragraphes.</p>\n<p><em>— Auteur</em></p></blockquote>",230 #"<div class="callout callout-warning"><span class="callout-icon">⚠️</span><div><p>Attention aux points suivants :</p>"#,231 "<details open><summary>Détails <strong>repliables</strong></summary><p>Contenu caché.</p>",232 "<table><tr><th><p>Poste</p></th>", "<td><p>Loyer | brut</p></td>", #"<td colspan="2"><figure><img src="/api/files/aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" alt="dans la table"></figure></td>"#,233 #"<figure><img src="/api/files/9d8c7b6a-5e4f-4d3c-8b2a-1f0e9d8c7b6a" alt="Façade nord"><figcaption>Photo du 12 mars</figcaption></figure>"#,234 #"<p class="file">📎 <a href="/api/files/11111111-2222-4333-8444-555555555555">rapport final.pdf</a> <small>(179 KB)</small></p>"#,235 "<pre><code class=\"language-python\">def f(x):\n return x * 2\n\nprint(f(21))</code></pre>", #"<div class="math-block"><code class="math">\int_0^1 x^2\,dx = \tfrac{1}{3}</code></div>"#,236 "<hr>", #"<p class="file">▶ <a href="https://www.youtube.com/watch?v=dQw4w9WgXcQ">https://www.youtube.com/watch?v=dQw4w9WgXcQ</a></p>"#,237 #"<div class="ai-block" title="Generated by claude-haiku"><p>Résumé généré.</p>"#, "<h3>Fin</h3>", "<p></p>",238 ##"<p class="footnote" id="fn-1"><sup>1</sup> Source : <a href="https://example.org/registre">registre foncier</a> .</p>"##] {239 #expect(html.contains(piece), "missing \(piece)")240 }241 }242243 @Test func textIsEscapedAndUrlsAreSafe() throws {244 let json = #"""245 {"type":"doc","content":[246 {"type":"paragraph","content":[{"type":"text","text":"<script>alert(\"x\")</script> & co"},247 {"type":"text","text":"js","marks":[{"type":"link","attrs":{"href":"javascript:alert(1)"}}]},248 {"type":"text","text":"rel","marks":[{"type":"link","attrs":{"href":"./a\"b"}}]},249 {"type":"text","text":"none","marks":[{"type":"link"}]},250 {"type":"mention","attrs":{"id":"<i>"}}]},251 {"type":"image","attrs":{"src":"data:text/html;base64,AAAA","alt":"\"><b>"}},252 {"type":"image","attrs":{"src":"data:image/PNG;base64,iVBOR","title":"<t>"}},253 {"type":"fileBlock","attrs":{"src":"ftp://x","size":100}},254 {"type":"codeBlock","attrs":{"language":"\"><x"},"content":[{"type":"text","text":"a < b && c > d"}]},255 {"type":"heading","attrs":{"level":9},"content":[{"type":"text","text":"h"}]},256 {"type":"orderedList","content":[{"type":"listItem","content":[{"type":"paragraph"}]}]},257 {"type":"callout","content":[{"type":"paragraph"}]},{"type":"widget","content":[{"type":"paragraph","content":[{"type":"text","text":"w"}]}]},{"type":"oddText","text":"<o>"}258 ]}259 """#260 let body = try NoteDocument(json: Data(json.utf8)).htmlBody()261 #expect(body == ##"""262 <p><script>alert("x")</script> & co<a href="#">js</a><a href="./a"b">rel</a><a href="#">none</a><span class="mention">@<i></span></p>263 <figure><img src="#" alt=""><b>"></figure>264 <figure><img src="data:image/PNG;base64,iVBOR" alt=""><figcaption><t></figcaption></figure>265 <p class="file">📎 <a href="#">file</a> <small>(0 KB)</small></p>266 <pre><code class="language-"><x">a < b && c > d</code></pre>267 <h6>h</h6>268 <ol start="1"><li><p></p></li></ol>269 <div class="callout callout-note"><span class="callout-icon">💡</span><div><p></p></div></div>270 <p>w</p>271 <p><o></p>272 """##)273 }274}275