Repository Public
1// SP Notes iOS2// File: Editor/SyntaxHighlighter.swift3// Description: Native single-pass syntax tokenizer for code blocks (comments, strings, numbers, keywords, types, calls) covering4// common languages — the tokenizer of the macOS editor (SPNotes/Editor/SyntaxHighlighter.swift), ported as is — and5// its iOS presentation: UTF-16 spans and token colours from the design tokens. Colours are never stored.6//7// Author: Simon-Pierre Boucher8// Contact: contact@spboucher.ai9import Foundation10import UIKit1112enum TokenKind: Sendable, Hashable { case plain, keyword, string, comment, number, type, function, attribute, punctuation }1314struct CodeToken: Sendable, Hashable {15 var text: String16 var kind: TokenKind17}1819enum SyntaxHighlighter {20 struct Grammar {21 var keywords: Set<String>22 var lineComments: [String] = ["//"]23 var blockComment: (String, String)? = ("/*", "*/")24 var stringDelimiters: [String] = ["\"", "'"]25 var multilineStrings: [String] = []26 var attributePrefix: Character? = nil27 var caseInsensitive = false28 var typesAreCapitalized = true29 }3031 private static let cLike: Set<String> = ["if", "else", "for", "while", "do", "switch", "case", "default", "break", "continue", "return", "struct", "enum", "union", "typedef", "static", "const", "void", "int", "long", "short", "char", "float", "double", "unsigned", "signed", "sizeof", "goto", "extern", "inline", "volatile", "true", "false", "NULL", "nullptr", "class", "public", "private", "protected", "virtual", "override", "template", "typename", "namespace", "using", "new", "delete", "this", "throw", "try", "catch", "auto", "bool", "operator", "constexpr", "noexcept", "include", "define", "ifdef", "ifndef", "endif", "pragma"]32 private static let js: Set<String> = ["const", "let", "var", "function", "return", "if", "else", "for", "while", "do", "switch", "case", "default", "break", "continue", "class", "extends", "new", "this", "super", "import", "export", "from", "as", "async", "await", "try", "catch", "finally", "throw", "typeof", "instanceof", "in", "of", "null", "undefined", "true", "false", "yield", "static", "get", "set", "delete", "void", "interface", "type", "enum", "implements", "public", "private", "protected", "readonly", "declare", "namespace", "abstract", "keyof", "never", "unknown", "any", "string", "number", "boolean", "satisfies"]33 private static let python: Set<String> = ["def", "class", "return", "if", "elif", "else", "for", "while", "break", "continue", "pass", "import", "from", "as", "try", "except", "finally", "raise", "with", "lambda", "yield", "global", "nonlocal", "assert", "del", "in", "not", "and", "or", "is", "None", "True", "False", "async", "await", "match", "case", "self"]34 private static let swift: Set<String> = ["let", "var", "func", "return", "if", "else", "guard", "for", "while", "repeat", "switch", "case", "default", "break", "continue", "struct", "class", "enum", "protocol", "extension", "actor", "import", "init", "deinit", "self", "Self", "super", "nil", "true", "false", "throw", "throws", "rethrows", "try", "catch", "do", "defer", "async", "await", "in", "is", "as", "where", "static", "private", "fileprivate", "internal", "public", "open", "final", "override", "mutating", "nonmutating", "lazy", "weak", "unowned", "some", "any", "typealias", "associatedtype", "subscript", "operator", "inout", "indirect", "convenience", "required", "nonisolated", "isolated", "consuming", "borrowing", "macro"]35 private static let rust: Set<String> = ["fn", "let", "mut", "const", "static", "struct", "enum", "trait", "impl", "for", "in", "while", "loop", "if", "else", "match", "return", "break", "continue", "use", "mod", "pub", "crate", "self", "Self", "super", "as", "where", "async", "await", "move", "ref", "dyn", "unsafe", "extern", "type", "true", "false", "Some", "None", "Ok", "Err"]36 private static let go: Set<String> = ["func", "var", "const", "type", "struct", "interface", "map", "chan", "package", "import", "return", "if", "else", "for", "range", "switch", "case", "default", "break", "continue", "go", "defer", "select", "fallthrough", "goto", "nil", "true", "false", "make", "new", "len", "cap", "append"]37 private static let java: Set<String> = ["public", "private", "protected", "class", "interface", "enum", "extends", "implements", "static", "final", "abstract", "void", "int", "long", "double", "float", "boolean", "char", "byte", "short", "new", "return", "if", "else", "for", "while", "do", "switch", "case", "default", "break", "continue", "try", "catch", "finally", "throw", "throws", "import", "package", "this", "super", "null", "true", "false", "var", "val", "fun", "when", "object", "companion", "data", "sealed", "override", "suspend", "record", "instanceof", "synchronized", "namespace", "using", "string", "readonly", "async", "await", "is", "as", "in", "out", "internal", "virtual", "struct", "get", "set"]38 private static let sql: Set<String> = ["select", "from", "where", "insert", "into", "values", "update", "set", "delete", "create", "table", "index", "view", "drop", "alter", "add", "column", "join", "left", "right", "inner", "outer", "full", "cross", "on", "as", "and", "or", "not", "null", "is", "in", "exists", "between", "like", "ilike", "group", "by", "order", "having", "limit", "offset", "union", "all", "distinct", "case", "when", "then", "else", "end", "primary", "key", "foreign", "references", "default", "constraint", "unique", "with", "returning", "begin", "commit", "rollback", "true", "false", "asc", "desc", "count", "sum", "avg", "min", "max", "coalesce", "cast", "over", "partition", "window", "if", "replace", "function", "trigger", "grant", "to"]39 private static let shell: Set<String> = ["if", "then", "else", "elif", "fi", "for", "in", "do", "done", "while", "until", "case", "esac", "function", "return", "exit", "export", "local", "readonly", "set", "unset", "echo", "cd", "source", "alias", "sudo", "true", "false", "trap", "shift", "eval", "exec", "test"]40 private static let ruby: Set<String> = ["def", "end", "class", "module", "if", "elsif", "else", "unless", "while", "until", "for", "in", "do", "return", "yield", "begin", "rescue", "ensure", "raise", "require", "include", "extend", "attr_accessor", "attr_reader", "self", "nil", "true", "false", "then", "case", "when", "and", "or", "not", "lambda", "proc", "puts", "super"]41 private static let rLang: Set<String> = ["function", "if", "else", "for", "while", "repeat", "break", "next", "return", "in", "TRUE", "FALSE", "NULL", "NA", "NaN", "Inf", "library", "require", "source"]4243 static func grammar(for language: String?) -> Grammar? {44 switch (language ?? "").lowercased() {45 case "swift": return Grammar(keywords: swift, multilineStrings: ["\"\"\""], attributePrefix: "@")46 case "js", "javascript", "jsx", "ts", "typescript", "tsx", "mjs", "cjs", "node": return Grammar(keywords: js, stringDelimiters: ["\"", "'", "`"], attributePrefix: "@")47 case "py", "python", "python3": return Grammar(keywords: python, lineComments: ["#"], blockComment: nil, multilineStrings: ["\"\"\"", "'''"], attributePrefix: "@")48 case "c", "h", "cpp", "c++", "cc", "hpp", "objc", "objective-c", "m", "mm", "metal", "glsl": return Grammar(keywords: cLike, attributePrefix: "#")49 case "rust", "rs": return Grammar(keywords: rust, stringDelimiters: ["\""], attributePrefix: "#")50 case "go", "golang": return Grammar(keywords: go, stringDelimiters: ["\"", "'", "`"])51 case "java", "kotlin", "kt", "scala", "cs", "csharp", "c#", "dart", "php", "groovy": return Grammar(keywords: java, attributePrefix: "@")52 case "sql", "psql", "postgres", "postgresql", "mysql", "sqlite", "plpgsql": return Grammar(keywords: sql, lineComments: ["--"], stringDelimiters: ["'"], caseInsensitive: true, typesAreCapitalized: false)53 case "sh", "bash", "zsh", "shell", "console", "fish", "dockerfile", "makefile", "make": return Grammar(keywords: shell, lineComments: ["#"], blockComment: nil, typesAreCapitalized: false)54 case "rb", "ruby": return Grammar(keywords: ruby, lineComments: ["#"], blockComment: nil, attributePrefix: "@")55 case "r": return Grammar(keywords: rLang, lineComments: ["#"], blockComment: nil, typesAreCapitalized: false)56 case "yaml", "yml", "toml", "ini", "conf", "env", "properties": return Grammar(keywords: ["true", "false", "null", "yes", "no", "on", "off"], lineComments: ["#"], blockComment: nil, typesAreCapitalized: false)57 case "json", "jsonc", "json5": return Grammar(keywords: ["true", "false", "null"], stringDelimiters: ["\""], typesAreCapitalized: false)58 case "html", "xml", "svg", "vue", "xhtml", "plist": return Grammar(keywords: [], lineComments: [], blockComment: ("<!--", "-->"), typesAreCapitalized: false)59 case "css", "scss", "less": return Grammar(keywords: ["important", "media", "import", "keyframes", "from", "to", "root", "var", "calc"], lineComments: [], attributePrefix: "@", typesAreCapitalized: false)60 case "tex", "latex": return Grammar(keywords: [], lineComments: ["%"], blockComment: nil, stringDelimiters: [], typesAreCapitalized: false)61 case "lua": return Grammar(keywords: ["local", "function", "end", "if", "then", "else", "elseif", "for", "while", "do", "repeat", "until", "return", "nil", "true", "false", "and", "or", "not", "in", "break"], lineComments: ["--"], blockComment: ("--[[", "]]"))62 case "", "text", "txt", "plain", "plaintext", "output", "log", "diff", "markdown", "md", "mermaid", "csv": return nil63 default: return Grammar(keywords: cLike.union(js).union(python), lineComments: ["//", "#"])64 }65 }6667 /// Splits `code` into coloured runs. Linear in the size of the input; unknown languages get a generic grammar and68 /// plain-text languages a single run.69 static func tokens(_ code: String, language: String?) -> [CodeToken] {70 guard let g = grammar(for: language), code.utf8.count < 400_000 else { return [CodeToken(text: code, kind: .plain)] }71 let s = Array(code); let n = s.count; var i = 072 var out: [CodeToken] = []; var plain = ""73 func flushPlain() { if !plain.isEmpty { out.append(CodeToken(text: plain, kind: .plain)); plain = "" } }74 func emit(_ range: Range<Int>, _ kind: TokenKind) { flushPlain(); out.append(CodeToken(text: String(s[range]), kind: kind)) }75 func starts(_ lit: String, at: Int) -> Bool { let a = Array(lit); guard at + a.count <= n else { return false }; for k in 0..<a.count where s[at + k] != a[k] { return false }; return true }76 let markup = g.blockComment?.0 == "<!--"77 while i < n {78 let c = s[i]79 if let bc = g.blockComment, starts(bc.0, at: i) {80 var j = i + bc.0.count; while j < n, !starts(bc.1, at: j) { j += 1 }81 let end = min(n, j + bc.1.count); emit(i..<end, .comment); i = end; continue82 }83 if let lc = g.lineComments.first(where: { starts($0, at: i) }) {84 // `#` only opens a comment at a word boundary (so that `a#b` or CSS colours inside values stay intact)85 if lc != "#" || i == 0 || s[i - 1].isWhitespace || s[i - 1] == ";" { var j = i; while j < n, s[j] != "\n" { j += 1 }; emit(i..<j, .comment); i = j; continue }86 }87 if let ms = g.multilineStrings.first(where: { starts($0, at: i) }) {88 var j = i + ms.count; while j < n, !starts(ms, at: j) { j += s[j] == "\\" ? 2 : 1 }89 let end = min(n, j + ms.count); emit(i..<end, .string); i = end; continue90 }91 if g.stringDelimiters.contains(String(c)) {92 var j = i + 193 while j < n, s[j] != c { if s[j] == "\\" { j += 1 } else if s[j] == "\n" && c != "`" { break }; j += 1 }94 let end = min(n, j + 1); emit(i..<end, .string); i = end; continue95 }96 if markup, c == "<" {97 var j = i + 1; if j < n, s[j] == "/" { j += 1 }98 let nameStart = j; while j < n, s[j].isLetter || s[j].isNumber || s[j] == "-" || s[j] == ":" { j += 1 }99 if j > nameStart { emit(i..<nameStart, .punctuation); emit(nameStart..<j, .keyword); i = j; continue }100 }101 if c.isNumber, i == 0 || !(s[i - 1].isLetter || s[i - 1] == "_") {102 var j = i + 1; while j < n, s[j].isNumber || s[j].isLetter || s[j] == "." || s[j] == "_" { j += 1 }103 emit(i..<j, .number); i = j; continue104 }105 if let ap = g.attributePrefix, c == ap, i + 1 < n, s[i + 1].isLetter {106 var j = i + 1; while j < n, s[j].isLetter || s[j].isNumber || s[j] == "_" || s[j] == "-" { j += 1 }107 emit(i..<j, .attribute); i = j; continue108 }109 if c.isLetter || c == "_" {110 var j = i + 1; while j < n, s[j].isLetter || s[j].isNumber || s[j] == "_" { j += 1 }111 let word = String(s[i..<j])112 let isKeyword = g.caseInsensitive ? g.keywords.contains(word.lowercased()) : g.keywords.contains(word)113 if isKeyword { emit(i..<j, .keyword) }114 else if markup { var k = j; while k < n, s[k] == " " { k += 1 }; if k < n, s[k] == "=" { emit(i..<j, .attribute) } else { plain += word } }115 else if j < n, s[j] == "(" { emit(i..<j, .function) }116 else if g.typesAreCapitalized, let f = word.first, f.isUppercase, word.count > 1 { emit(i..<j, .type) }117 else { plain += word }118 i = j; continue119 }120 plain.append(c); i += 1121 }122 flushPlain()123 return out124 }125}126127extension SyntaxHighlighter {128 struct Span: Hashable { var range: NSRange; var kind: TokenKind }129130 /// The coloured runs of `code` as UTF-16 ranges (plain runs are left out).131 static func spans(_ code: String, language: String?) -> [Span] {132 var out: [Span] = []133 var location = 0134 for token in tokens(code, language: language) {135 let length = token.text.utf16.count136 if token.kind != .plain, length > 0 { out.append(Span(range: NSRange(location: location, length: length), kind: token.kind)) }137 location += length138 }139 return out140 }141142 /// Token colours from the Note Visual System tokens (light and dark). Violet is reserved for AI and not used here.143 static func colour(for kind: TokenKind) -> UIColor? {144 switch kind {145 case .plain: nil146 case .keyword: NVUIColor.dangerText147 case .string: NVUIColor.successText148 case .comment: NVUIColor.fgMuted149 case .number: NVUIColor.warningText150 case .type: NVUIColor.accent151 case .function: NVUIColor.accent.withAlphaComponent(0.78)152 case .attribute: NVUIColor.warningText153 case .punctuation: NVUIColor.fgMuted154 }155 }156157 /// Languages offered by the code-block language menu (any other value written by another editor is kept).158 static let menuLanguages = ["plaintext", "swift", "python", "javascript", "typescript", "json", "sql", "bash", "c", "cpp", "rust", "go", "java", "kotlin", "ruby", "r", "html", "css", "yaml", "latex", "lua"]159}160