Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: SPChat/Tests/CoreTests/MarkdownTests.swift3// Description: Markdown → render model (tables, code, LaTeX protection), streaming chunk stability, link policy, syntax tokens, file sniffing, redaction, export.4import Foundation5import Testing6@testable import SPChatCore78@Suite struct MarkdownTests {9 private let p = MarkdownParser(capacity: 100)1011 @Test func blocks() {12 let b = p.parse("# Titre\n\nUn **gras** et *italique* avec `code` et [lien](https://example.org).\n\n- a\n- [x] fait\n\n1. un\n2. deux\n\n> cité\n\n---\n\n```swift\nlet x = 1\n```\n")13 #expect(b.count == 7)14 #expect(b[0] == .heading(level: 1, [.text("Titre")]))15 if case .paragraph(let i) = b[1] { #expect(i.contains(.strong([.text("gras")]))); #expect(i.contains(.code("code"))); #expect(i.contains(.link(destination: "https://example.org", children: [.text("lien")]))) } else { Issue.record("paragraph") }16 if case .list(false, _, let items) = b[2] { #expect(items.count == 2); #expect(items[1].checkbox == true) } else { Issue.record("list") }17 if case .list(true, let start, let items) = b[3] { #expect(start == 1); #expect(items.count == 2) } else { Issue.record("ordered") }18 #expect(b[5] == .rule); #expect(b[6] == .code(language: "swift", text: "let x = 1"))19 }2021 @Test func tables() {22 let b = p.parse("| Modèle | Prix |\n|:--|--:|\n| A | 1 $ |\n| B | 2 $ |")23 guard case .table(let t) = b.first else { Issue.record("table"); return }24 #expect(t.header.map { $0.map(\.plainText).joined() } == ["Modèle", "Prix"]); #expect(t.rows.count == 2); #expect(t.alignments == [.leading, .trailing]); #expect(t.columnCount == 2)25 #expect(b[0].plainText.contains("A\t1 $"))26 }2728 @Test func mathIsProtectedFromMarkdown() {29 let b = p.parse(#"La variance $\sigma_t^2 = \omega + \alpha \epsilon_{t-1}^2$ et \(a_i * b_i\) en ligne."# + "\n\n$$\n\\sum_{i=1}^{n} x_i * y_i\n$$\n\n\\[ E = mc^2 \\]")30 guard case .paragraph(let i) = b[0] else { Issue.record("paragraph"); return }31 #expect(i.contains(.math(#"\sigma_t^2 = \omega + \alpha \epsilon_{t-1}^2"#, display: false))); #expect(i.contains(.math("a_i * b_i", display: false)))32 #expect(b[1] == .math(#"\sum_{i=1}^{n} x_i * y_i"#)); #expect(b[2] == .math("E = mc^2"))33 }3435 @Test func currencyIsNotMath() {36 let b = p.parse("Ça coûte 5 $ ou $5 et $10, pas plus. Un prix de 3$ et 4$ aussi.")37 if case .paragraph(let i) = b[0] { #expect(!i.contains { if case .math = $0 { return true }; return false }) } else { Issue.record("paragraph") }38 }3940 @Test func dollarsInsideCodeStayLiteral() {41 let b = p.parse("Inline `echo $HOME and $PATH` ok.\n\n```bash\necho $A $B\nx=$(ls)\n```\n\nDisplay inside a paragraph: avant $$x^2$$ après")42 if case .paragraph(let i) = b[0] { #expect(i.contains(.code("echo $HOME and $PATH"))) } else { Issue.record("inline code") }43 #expect(b[1] == .code(language: "bash", text: "echo $A $B\nx=$(ls)"))44 #expect(Array(b[2...]) == [.paragraph([.text("Display inside a paragraph: avant ")]), .math("x^2"), .paragraph([.text(" après")])])45 }4647 @Test func rawHTMLIsNeverInterpreted() {48 let b = p.parse("<script>alert(1)</script>\n\nTexte <img src=x onerror=alert(1)> fin")49 #expect(b[0] == .html("<script>alert(1)</script>"))50 if case .paragraph(let i) = b[1] { #expect(i.map(\.plainText).joined().contains("<img src=x onerror=alert(1)>")) } else { Issue.record("paragraph") }51 }5253 @Test func linkPolicy() {54 #expect(LinkPolicy.safeURL("https://example.org/a?b=1") != nil); #expect(LinkPolicy.safeURL("mailto:a@b.c") != nil)55 for bad in ["javascript:alert(1)", "file:///etc/passwd", "spchat://delete-all", "data:text/html,x", "https://", "x-apple.systempreferences:x", " JAVASCRIPT:alert(1)"] { #expect(LinkPolicy.safeURL(bad) == nil, "\(bad)") }56 }5758 @Test func chunkingKeepsFencesMathAndListsTogether() {59 let src = "Intro\n\n```python\na = 1\n\nb = 2\n```\n\n$$\nx\n\ny\n$$\n\n- un\n- deux\n\nFin"60 let chunks = MarkdownParser.split(src)61 #expect(chunks == ["Intro", "```python\na = 1\n\nb = 2\n```", "$$\nx\n\ny\n$$", "- un\n- deux", "Fin"])62 #expect(p.chunks(src).map(\.id) == [0, 1, 2, 3, 4])63 }6465 @Test func streamingOnlyTouchesTheLastChunk() {66 let full = "Premier paragraphe assez long.\n\n## Section\n\nDeuxième paragraphe avec du **gras**.\n\n```swift\nfunc f() {\n print(1)\n}\n```\n\nConclusion finale."67 var previous: [MDChunk] = []; var rewrites = 068 for end in stride(from: 1, through: full.count, by: 3) {69 let now = p.chunks(String(full.prefix(end)))70 // every chunk except the last must be identical to what it was at the previous step71 if previous.count > 1, now.count >= previous.count { for k in 0..<(previous.count - 1) where now[k] != previous[k] { rewrites += 1 } }72 previous = now73 }74 #expect(rewrites == 0)75 #expect(p.chunks(full).flatMap(\.blocks) == p.parse(full)) // chunked parsing gives the same document76 }7778 @Test func unterminatedFenceWhileStreaming() {79 let b = p.chunks("Voici :\n\n```js\nconst a = 1;\nconst b")80 #expect(b.count == 2); #expect(b[1].blocks == [.code(language: "js", text: "const a = 1;\nconst b")])81 }8283 @Test func longDocumentParsesFast() {84 let para = "Un paragraphe avec **gras**, `code`, $x_i^2$ et [lien](https://example.org). " + String(repeating: "Du texte courant. ", count: 12)85 let doc = (0..<1500).map { "\(para)\($0)" }.joined(separator: "\n\n")86 let fresh = MarkdownParser(); let clock = ContinuousClock(); var n = 087 let cold = clock.measure { n = fresh.chunks(doc).count }88 let warm = clock.measure { _ = fresh.chunks(doc + "\n\nUn dernier.") }89 #expect(n == 1500); #expect(cold < .seconds(4)); #expect(warm < cold) // the second pass re-uses 1500 cached chunks90 }91}9293@Suite struct SyntaxHighlighterTests {94 private func kinds(_ code: String, _ lang: String?) -> [(String, TokenKind)] { SyntaxHighlighter.tokens(code, language: lang).filter { $0.kind != .plain }.map { ($0.text, $0.kind) } }9596 @Test func swiftTokens() {97 let k = kinds("@MainActor func greet(_ name: String) -> Int { // salut\n let s = \"a\\\"b\"; return 42 }", "swift")98 #expect(k.contains { $0 == ("@MainActor", .attribute) }); #expect(k.contains { $0 == ("func", .keyword) }); #expect(k.contains { $0 == ("greet", .function) })99 #expect(k.contains { $0 == ("String", .type) }); #expect(k.contains { $0 == ("// salut", .comment) }); #expect(k.contains { $0 == ("\"a\\\"b\"", .string) }); #expect(k.contains { $0 == ("42", .number) })100 }101102 @Test func pythonSqlShellAndHtml() {103 #expect(kinds("def f(x):\n \"\"\"doc\n string\"\"\"\n return x # fin", "python").contains { $0 == ("\"\"\"doc\n string\"\"\"", .string) })104 #expect(kinds("SELECT id FROM t WHERE a = 'x' -- c", "sql").map(\.0) == ["SELECT", "FROM", "WHERE", "'x'", "-- c"])105 #expect(kinds("echo $HOME # commentaire", "bash").contains { $0 == ("# commentaire", .comment) })106 let html = kinds("<div class=\"a\"><!-- c --></div>", "html")107 #expect(html.contains { $0 == ("div", .keyword) }); #expect(html.contains { $0 == ("class", .attribute) }); #expect(html.contains { $0 == ("<!-- c -->", .comment) })108 }109110 @Test func outputAlwaysReassemblesTheSource() {111 let samples = [("let a = \"unterminated", "swift"), ("/* never closed", "c"), ("x = '''open", "python"), ("😀 émoji // ok", "js"), ("", "swift"), ("plain text", nil), ("a\tb\r\nc", "go")]112 for (code, lang) in samples { #expect(SyntaxHighlighter.tokens(code, language: lang).map(\.text).joined() == code, "\(code)") }113 }114115 @Test func largeBlockIsLinear() {116 let code = String(repeating: "func f(a: Int) -> String { return \"x\" } // c\n", count: 4000)117 let clock = ContinuousClock(); var n = 0118 let t = clock.measure { n = SyntaxHighlighter.tokens(code, language: "swift").count }119 #expect(n > 4000); #expect(t < .seconds(3))120 }121}122123@Suite struct SecurityTests {124 @Test func sniffsRealContent() {125 #expect(FileValidator.sniffMime(Data("%PDF-1.7 ...".utf8), filename: "a.txt") == "application/pdf") // content wins over the extension126 #expect(FileValidator.sniffMime(Data([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0]), filename: "x.png") == "image/png")127 #expect(FileValidator.sniffMime(Data([0xff, 0xd8, 0xff, 0xe0, 0, 0]), filename: "x.jpg") == "image/jpeg")128 #expect(FileValidator.sniffMime(Data("bonjour, é".utf8), filename: "note.md") == "text/markdown"); #expect(FileValidator.sniffMime(Data("a,b\n1,2".utf8), filename: "d.csv") == "text/csv")129 #expect(FileValidator.sniffMime(Data([0x4d, 0x5a, 0x90, 0x00]), filename: "virus.txt.exe") == nil) // unknown binary130 #expect(FileValidator.sniffMime(Data([0x4d, 0x5a, 0x00, 0x01, 0x00]), filename: "fake.txt") == nil) // binary disguised as text (NUL byte)131 #expect(FileValidator.sniffMime(Data([0x50, 0x4b, 0x03, 0x04] + Array("....word/document.xml....".utf8)), filename: "a.docx")?.contains("wordprocessingml") == true)132 #expect(FileValidator.sniffMime(Data([0x50, 0x4b, 0x03, 0x04] + Array("random.zip content".utf8)), filename: "a.zip") == nil)133 #expect(FileValidator.sniffMime(Data([0, 0, 0, 0x18] + Array("ftypheic".utf8) + [0, 0, 0, 0]), filename: "IMG.HEIC") == "image/heic")134 }135136 @Test func validation() {137 #expect(FileValidator.validate(data: Data(), filename: "a.txt", maxUploadMb: 50) == .failure(.empty("a.txt")))138 #expect(FileValidator.validate(data: Data(count: 2 * 1024 * 1024), filename: "a.txt", maxUploadMb: 1) == .failure(.tooLarge("a.txt", limitMb: 1)))139 #expect(FileValidator.validate(data: Data([1, 2, 3]), filename: "a.bin", maxUploadMb: 50) == .failure(.unsupported("a.bin")))140 #expect(FileValidator.validate(data: Data("ok".utf8), filename: "a.txt", maxUploadMb: 50) == .success("text/plain"))141 #expect(FileValidator.sanitizedName("../../etc/pass\"wd\n") == "....etcpasswd"); #expect(FileValidator.sanitizedName("\n") == "fichier")142 }143144 @Test func redaction() {145 let token = String(repeating: "Ab3_", count: 10) + "xyz"146 let s = Redactor.redact("Cookie: __Host-csa_session=\(token); x\nappel avec sk-abcdefghijklmnopqrstuvwxyz et code ABCD-EFGH-JKMN-PQRS-TVWX fin")147 #expect(!s.contains(token)); #expect(!s.contains("sk-abcdefghijklmnop")); #expect(!s.contains("ABCD-EFGH-JKMN-PQRS-TVWX")); #expect(s.contains("fin"))148 }149150 @Test func keychainRoundTrip() throws {151 let store = KeychainStore(service: "ai.spboucher.spchat.tests.\(UUID().uuidString)")152 #expect(store.read("k") == nil)153 do { try store.write("secret-1", for: "k") } catch { return } // no keychain in this environment (headless CI): nothing to verify154 #expect(store.read("k") == "secret-1"); try store.write("secret-2", for: "k"); #expect(store.read("k") == "secret-2")155 store.delete("k"); #expect(store.read("k") == nil)156 }157}158159@Suite struct ExportTests {160 @Test func markdownAndJson() throws {161 let c = Conversation(id: "c1", title: "Plan: été/2027 ?", createdAt: Date(timeIntervalSince1970: 0))162 let u = Message(id: "u", conversationId: "c1", role: .user, parts: [.file(attachmentId: "att", mime: "application/pdf", name: "plan.pdf", size: 10, pages: 2), .text("Résume.")])163 let old = Message(id: "a0", conversationId: "c1", parentId: "u", role: .assistant, parts: [.text("ancienne")], active: false)164 let a = Message(id: "a", conversationId: "c1", parentId: "u", role: .assistant, parts: [.reasoning(text: "secret", summarized: true), .toolCall(ToolCall(id: "t", name: "search_documents", input: ["q": "plan"], status: .ok, output: "long output")), .text("Voici.")],165 modelId: "claude-haiku-5-5", usage: TokenUsage(input: 10, output: 5), costUsd: 0.0012, latencyMs: 300, status: .incomplete, error: "coupé")166 let d = ConversationDetail(conversation: c, messages: [u, old, a])167 let md = ConversationExporter.markdown(d, now: Date(timeIntervalSince1970: 1_800_000_000))168 #expect(md.hasPrefix("# Plan: été/2027 ?\n")); #expect(md.contains("## Vous")); #expect(md.contains("## Assistant (claude-haiku-5-5)")); #expect(md.contains("> Pièce jointe : plan.pdf"))169 #expect(md.contains("> Outil search_documents(")); #expect(!md.contains("long output")); #expect(!md.contains("secret")); #expect(!md.contains("ancienne"))170 #expect(md.contains("> État : incomplete — coupé")); #expect(md.contains("10 → 5 jetons · 0.0012 $ (estimation) · 300 ms"))171 let j = JSONValue.parse(ConversationExporter.json(d))!172 #expect(j["format"]?.stringValue == "chat-spboucher-ai/conversation"); #expect(j["messages"]?.arrayValue?.count == 2)173 #expect(JSONValue.parse(ConversationExporter.json(d, includeInactiveBranches: true))!["messages"]?.arrayValue?.count == 3)174 #expect(ConversationExporter.fileName(title: "Plan: été/2027 ?", now: Date(timeIntervalSince1970: 1_800_000_000)) == "Conversation - Plan été 2027 - 2027-01-15")175 }176}177