Repository Public
1// Author: Simon-Pierre Boucher — contact@spboucher.ai2// File: Packages/LocalChatUI/Sources/Markdown/SyntaxHighlighter.swift3// Description: Native single-pass syntax tokenizer for code blocks (comments, strings, numbers, keywords, types, calls) covering common languages.4import Foundation56public enum TokenKind: Sendable, Hashable { case plain, keyword, string, comment, number, type, function, attribute, punctuation }78public struct CodeToken: Sendable, Hashable {9 public var text: String10 public var kind: TokenKind11}1213public enum SyntaxHighlighter {14 struct Grammar {15 var keywords: Set<String>16 var lineComments: [String] = ["//"]17 var blockComment: (String, String)? = ("/*", "*/")18 var stringDelimiters: [String] = ["\"", "'"]19 var multilineStrings: [String] = []20 var attributePrefix: Character? = nil21 var caseInsensitive = false22 var typesAreCapitalized = true23 }2425 private static let cLike: Set<String> = ["if", "else", "for", "while", "do", "switch", "case", "default", "break", "continue", "return", "struct", "enum", "union", "typedef", "static", "const", "void", "int", "long", "short", "char", "float", "double", "unsigned", "signed", "sizeof", "goto", "extern", "inline", "volatile", "true", "false", "NULL", "nullptr", "class", "public", "private", "protected", "virtual", "override", "template", "typename", "namespace", "using", "new", "delete", "this", "throw", "try", "catch", "auto", "bool", "operator", "constexpr", "noexcept", "include", "define", "ifdef", "ifndef", "endif", "pragma"]26 private static let js: Set<String> = ["const", "let", "var", "function", "return", "if", "else", "for", "while", "do", "switch", "case", "default", "break", "continue", "class", "extends", "new", "this", "super", "import", "export", "from", "as", "async", "await", "try", "catch", "finally", "throw", "typeof", "instanceof", "in", "of", "null", "undefined", "true", "false", "yield", "static", "get", "set", "delete", "void", "interface", "type", "enum", "implements", "public", "private", "protected", "readonly", "declare", "namespace", "abstract", "keyof", "never", "unknown", "any", "string", "number", "boolean", "satisfies"]27 private static let python: Set<String> = ["def", "class", "return", "if", "elif", "else", "for", "while", "break", "continue", "pass", "import", "from", "as", "try", "except", "finally", "raise", "with", "lambda", "yield", "global", "nonlocal", "assert", "del", "in", "not", "and", "or", "is", "None", "True", "False", "async", "await", "match", "case", "self"]28 private static let swift: Set<String> = ["let", "var", "func", "return", "if", "else", "guard", "for", "while", "repeat", "switch", "case", "default", "break", "continue", "struct", "class", "enum", "protocol", "extension", "actor", "import", "init", "deinit", "self", "Self", "super", "nil", "true", "false", "throw", "throws", "rethrows", "try", "catch", "do", "defer", "async", "await", "in", "is", "as", "where", "static", "private", "fileprivate", "internal", "public", "open", "final", "override", "mutating", "nonmutating", "lazy", "weak", "unowned", "some", "any", "typealias", "associatedtype", "subscript", "operator", "inout", "indirect", "convenience", "required", "nonisolated", "isolated", "consuming", "borrowing", "macro"]29 private static let rust: Set<String> = ["fn", "let", "mut", "const", "static", "struct", "enum", "trait", "impl", "for", "in", "while", "loop", "if", "else", "match", "return", "break", "continue", "use", "mod", "pub", "crate", "self", "Self", "super", "as", "where", "async", "await", "move", "ref", "dyn", "unsafe", "extern", "type", "true", "false", "Some", "None", "Ok", "Err"]30 private static let go: Set<String> = ["func", "var", "const", "type", "struct", "interface", "map", "chan", "package", "import", "return", "if", "else", "for", "range", "switch", "case", "default", "break", "continue", "go", "defer", "select", "fallthrough", "goto", "nil", "true", "false", "make", "new", "len", "cap", "append"]31 private static let java: Set<String> = ["public", "private", "protected", "class", "interface", "enum", "extends", "implements", "static", "final", "abstract", "void", "int", "long", "double", "float", "boolean", "char", "byte", "short", "new", "return", "if", "else", "for", "while", "do", "switch", "case", "default", "break", "continue", "try", "catch", "finally", "throw", "throws", "import", "package", "this", "super", "null", "true", "false", "var", "val", "fun", "when", "object", "companion", "data", "sealed", "override", "suspend", "record", "instanceof", "synchronized", "namespace", "using", "string", "readonly", "async", "await", "is", "as", "in", "out", "internal", "virtual", "struct", "get", "set"]32 private static let sql: Set<String> = ["select", "from", "where", "insert", "into", "values", "update", "set", "delete", "create", "table", "index", "view", "drop", "alter", "add", "column", "join", "left", "right", "inner", "outer", "full", "cross", "on", "as", "and", "or", "not", "null", "is", "in", "exists", "between", "like", "ilike", "group", "by", "order", "having", "limit", "offset", "union", "all", "distinct", "case", "when", "then", "else", "end", "primary", "key", "foreign", "references", "default", "constraint", "unique", "with", "returning", "begin", "commit", "rollback", "true", "false", "asc", "desc", "count", "sum", "avg", "min", "max", "coalesce", "cast", "over", "partition", "window", "if", "replace", "function", "trigger", "grant", "to"]33 private static let shell: Set<String> = ["if", "then", "else", "elif", "fi", "for", "in", "do", "done", "while", "until", "case", "esac", "function", "return", "exit", "export", "local", "readonly", "set", "unset", "echo", "cd", "source", "alias", "sudo", "true", "false", "trap", "shift", "eval", "exec", "test"]34 private static let ruby: Set<String> = ["def", "end", "class", "module", "if", "elsif", "else", "unless", "while", "until", "for", "in", "do", "return", "yield", "begin", "rescue", "ensure", "raise", "require", "include", "extend", "attr_accessor", "attr_reader", "self", "nil", "true", "false", "then", "case", "when", "and", "or", "not", "lambda", "proc", "puts", "super"]35 private static let rLang: Set<String> = ["function", "if", "else", "for", "while", "repeat", "break", "next", "return", "in", "TRUE", "FALSE", "NULL", "NA", "NaN", "Inf", "library", "require", "source"]3637 static func grammar(for language: String?) -> Grammar? {38 switch (language ?? "").lowercased() {39 case "swift": return Grammar(keywords: swift, multilineStrings: ["\"\"\""], attributePrefix: "@")40 case "js", "javascript", "jsx", "ts", "typescript", "tsx", "mjs", "cjs", "node": return Grammar(keywords: js, stringDelimiters: ["\"", "'", "`"], attributePrefix: "@")41 case "py", "python", "python3": return Grammar(keywords: python, lineComments: ["#"], blockComment: nil, multilineStrings: ["\"\"\"", "'''"], attributePrefix: "@")42 case "c", "h", "cpp", "c++", "cc", "hpp", "objc", "objective-c", "m", "mm", "metal", "glsl": return Grammar(keywords: cLike, attributePrefix: "#")43 case "rust", "rs": return Grammar(keywords: rust, stringDelimiters: ["\""], attributePrefix: "#")44 case "go", "golang": return Grammar(keywords: go, stringDelimiters: ["\"", "'", "`"])45 case "java", "kotlin", "kt", "scala", "cs", "csharp", "c#", "dart", "php", "groovy": return Grammar(keywords: java, attributePrefix: "@")46 case "sql", "psql", "postgres", "postgresql", "mysql", "sqlite", "plpgsql": return Grammar(keywords: sql, lineComments: ["--"], stringDelimiters: ["'"], caseInsensitive: true, typesAreCapitalized: false)47 case "sh", "bash", "zsh", "shell", "console", "fish", "dockerfile", "makefile", "make": return Grammar(keywords: shell, lineComments: ["#"], blockComment: nil, typesAreCapitalized: false)48 case "rb", "ruby": return Grammar(keywords: ruby, lineComments: ["#"], blockComment: nil, attributePrefix: "@")49 case "r": return Grammar(keywords: rLang, lineComments: ["#"], blockComment: nil, typesAreCapitalized: false)50 case "yaml", "yml", "toml", "ini", "conf", "env", "properties": return Grammar(keywords: ["true", "false", "null", "yes", "no", "on", "off"], lineComments: ["#"], blockComment: nil, typesAreCapitalized: false)51 case "json", "jsonc", "json5": return Grammar(keywords: ["true", "false", "null"], stringDelimiters: ["\""], typesAreCapitalized: false)52 case "html", "xml", "svg", "vue", "xhtml", "plist": return Grammar(keywords: [], lineComments: [], blockComment: ("<!--", "-->"), typesAreCapitalized: false)53 case "css", "scss", "less": return Grammar(keywords: ["important", "media", "import", "keyframes", "from", "to", "root", "var", "calc"], lineComments: [], attributePrefix: "@", typesAreCapitalized: false)54 case "tex", "latex": return Grammar(keywords: [], lineComments: ["%"], blockComment: nil, stringDelimiters: [], typesAreCapitalized: false)55 case "lua": return Grammar(keywords: ["local", "function", "end", "if", "then", "else", "elseif", "for", "while", "do", "repeat", "until", "return", "nil", "true", "false", "and", "or", "not", "in", "break"], lineComments: ["--"], blockComment: ("--[[", "]]"))56 case "", "text", "txt", "plain", "plaintext", "output", "log", "diff", "markdown", "md", "mermaid", "csv": return nil57 default: return Grammar(keywords: cLike.union(js).union(python), lineComments: ["//", "#"])58 }59 }6061 /// Splits `code` into coloured runs. Linear in the size of the input; unknown languages get a generic grammar and62 /// plain-text languages a single run.63 public static func tokens(_ code: String, language: String?) -> [CodeToken] {64 guard let g = grammar(for: language), code.utf8.count < 400_000 else { return [CodeToken(text: code, kind: .plain)] }65 let s = Array(code); let n = s.count; var i = 066 var out: [CodeToken] = []; var plain = ""67 func flushPlain() { if !plain.isEmpty { out.append(CodeToken(text: plain, kind: .plain)); plain = "" } }68 func emit(_ range: Range<Int>, _ kind: TokenKind) { flushPlain(); out.append(CodeToken(text: String(s[range]), kind: kind)) }69 func starts(_ lit: String, at: Int) -> Bool { let a = Array(lit); guard at + a.count <= n else { return false }; for k in 0..<a.count where s[at + k] != a[k] { return false }; return true }70 let markup = g.blockComment?.0 == "<!--"71 while i < n {72 let c = s[i]73 if let bc = g.blockComment, starts(bc.0, at: i) {74 var j = i + bc.0.count; while j < n, !starts(bc.1, at: j) { j += 1 }75 let end = min(n, j + bc.1.count); emit(i..<end, .comment); i = end; continue76 }77 if let lc = g.lineComments.first(where: { starts($0, at: i) }) {78 // `#` only opens a comment at a word boundary (so that `a#b` or CSS colours inside values stay intact)79 if lc != "#" || i == 0 || s[i - 1].isWhitespace || s[i - 1] == ";" { var j = i; while j < n, s[j] != "\n" { j += 1 }; emit(i..<j, .comment); i = j; continue }80 }81 if let ms = g.multilineStrings.first(where: { starts($0, at: i) }) {82 var j = i + ms.count; while j < n, !starts(ms, at: j) { j += s[j] == "\\" ? 2 : 1 }83 let end = min(n, j + ms.count); emit(i..<end, .string); i = end; continue84 }85 if g.stringDelimiters.contains(String(c)) {86 var j = i + 187 while j < n, s[j] != c { if s[j] == "\\" { j += 1 } else if s[j] == "\n" && c != "`" { break }; j += 1 }88 let end = min(n, j + 1); emit(i..<end, .string); i = end; continue89 }90 if markup, c == "<" {91 var j = i + 1; if j < n, s[j] == "/" { j += 1 }92 let nameStart = j; while j < n, s[j].isLetter || s[j].isNumber || s[j] == "-" || s[j] == ":" { j += 1 }93 if j > nameStart { emit(i..<nameStart, .punctuation); emit(nameStart..<j, .keyword); i = j; continue }94 }95 if c.isNumber, i == 0 || !(s[i - 1].isLetter || s[i - 1] == "_") {96 var j = i + 1; while j < n, s[j].isNumber || s[j].isLetter || s[j] == "." || s[j] == "_" { j += 1 }97 emit(i..<j, .number); i = j; continue98 }99 if let ap = g.attributePrefix, c == ap, i + 1 < n, s[i + 1].isLetter {100 var j = i + 1; while j < n, s[j].isLetter || s[j].isNumber || s[j] == "_" || s[j] == "-" { j += 1 }101 emit(i..<j, .attribute); i = j; continue102 }103 if c.isLetter || c == "_" {104 var j = i + 1; while j < n, s[j].isLetter || s[j].isNumber || s[j] == "_" { j += 1 }105 let word = String(s[i..<j])106 let isKeyword = g.caseInsensitive ? g.keywords.contains(word.lowercased()) : g.keywords.contains(word)107 if isKeyword { emit(i..<j, .keyword) }108 else if markup { var k = j; while k < n, s[k] == " " { k += 1 }; if k < n, s[k] == "=" { emit(i..<j, .attribute) } else { plain += word } }109 else if j < n, s[j] == "(" { emit(i..<j, .function) }110 else if g.typesAreCapitalized, let f = word.first, f.isUppercase, word.count > 1 { emit(i..<j, .type) }111 else { plain += word }112 i = j; continue113 }114 plain.append(c); i += 1115 }116 flushPlain()117 return out118 }119}120