Repository Public
1// ORIX DJ2// File: Packages/ORIXDJCore/Sources/ORIXScript/Lexer.swift3// Description: ORIX Script lexer: source text to tokens, including unit literals (-12dB, 20Hz, 4bars, 1/4beat, 50%).4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation89public enum TokenKind: Equatable, Sendable {10 case identifier(String)11 case number(Double)12 /// A number with a unit, already converted to a value.13 case literal(Value)14 case string(String)15 case symbol(String)16 case newline17 case eof18}1920public struct Token: Equatable, Sendable {21 public var kind: TokenKind22 public var location: SourceLocation23 public var length: Int24 /// Offset range in the source (UTF-16 units), for the editor's syntax colouring.25 public var utf16Range: Range<Int>26}2728public enum TokenClass: Sendable { case keyword, identifier, number, string, comment, symbol, root }2930/// A coloured span for the editor. Produced even for text that does not parse.31public struct HighlightSpan: Sendable, Equatable {32 public var range: Range<Int>33 public var tokenClass: TokenClass34}3536public struct Lexer {37 private let chars: [Character]38 private let utf16Offsets: [Int]39 private var i = 040 private var line = 141 private var column = 142 public private(set) var diagnostics: [Diagnostic] = []43 public private(set) var comments: [Range<Int>] = []4445 public init(_ source: String) {46 chars = Array(source)47 var offsets: [Int] = []; offsets.reserveCapacity(chars.count + 1)48 var o = 049 for c in chars { offsets.append(o); o += c.utf16.count }50 offsets.append(o)51 utf16Offsets = offsets52 }5354 static let units: Set<String> = ["Hz", "kHz", "dB", "beat", "beats", "bar", "bars", "s", "sec", "ms", "min", "bpm", "st"]55 static let wordUnits: Set<String> = ["beat", "beats", "bar", "bars"]5657 static func unitValue(_ x: Double, _ unit: String) -> Value? {58 switch unit {59 case "Hz": .frequency(x)60 case "kHz": .frequency(x * 1000)61 case "dB": .gain(x)62 case "beat", "beats": .duration(.beats(x))63 case "bar", "bars": .duration(.beats(x * ORIXScriptLanguage.beatsPerBar))64 case "s", "sec": .duration(.seconds(x))65 case "ms": .duration(.seconds(x / 1000))66 case "min": .duration(.seconds(x * 60))67 case "bpm", "st": .number(x)68 case "%": .number(x / 100)69 default: nil70 }71 }7273 private func peek(_ k: Int = 0) -> Character? { i + k < chars.count ? chars[i + k] : nil }7475 private mutating func advance() {76 if chars[i] == "\n" { line += 1; column = 1 } else { column += 1 }77 i += 178 }7980 private static func isIdentStart(_ c: Character) -> Bool { c.isLetter || c == "_" }81 private static func isIdentPart(_ c: Character) -> Bool { c.isLetter || c.isNumber || c == "_" }82 private static func isDigit(_ c: Character?) -> Bool { c.map { $0.isASCII && $0.isNumber } ?? false }8384 public mutating func tokenize() -> [Token] {85 var tokens: [Token] = []86 let offsets = utf16Offsets87 let make: (TokenKind, Int, SourceLocation, Int) -> Token = { kind, start, loc, end in88 Token(kind: kind, location: loc, length: end - start, utf16Range: offsets[start] ..< offsets[end])89 }90 while let c = peek() {91 let start = i; let loc = SourceLocation(line: line, column: column)92 if c == "\n" { advance(); tokens.append(make(.newline, start, loc, i)); continue }93 if c == ";" { advance(); tokens.append(make(.newline, start, loc, i)); continue }94 if c.isWhitespace { advance(); continue }95 if c == "/", peek(1) == "/" {96 while let d = peek(), d != "\n" { _ = d; advance() }97 comments.append(utf16Offsets[start] ..< utf16Offsets[i]); continue98 }99 if c == "/", peek(1) == "*" {100 advance(); advance()101 var closed = false102 while peek() != nil {103 if peek() == "*", peek(1) == "/" { advance(); advance(); closed = true; break }104 advance()105 }106 if !closed { diagnostics.append(Diagnostic(code: "E-LEX", "Unterminated block comment.", at: loc, length: 2)) }107 comments.append(utf16Offsets[start] ..< utf16Offsets[i]); continue108 }109 if Self.isDigit(c) || (c == "." && Self.isDigit(peek(1))) {110 tokens.append(lexNumber(start, loc, make)); continue111 }112 if Self.isIdentStart(c) {113 while let d = peek(), Self.isIdentPart(d) { advance() }114 tokens.append(make(.identifier(String(chars[start ..< i])), start, loc, i)); continue115 }116 if c == "\"" {117 advance()118 var text = ""; var closed = false119 while let d = peek() {120 if d == "\n" { break }121 if d == "\\" {122 advance()123 switch peek() {124 case "n": text.append("\n")125 case "t": text.append("\t")126 case let e?: text.append(e)127 case nil: break128 }129 if peek() != nil { advance() }130 continue131 }132 if d == "\"" { advance(); closed = true; break }133 text.append(d); advance()134 }135 if !closed { diagnostics.append(Diagnostic(code: "E-LEX", "Unterminated string.", at: loc, length: i - start)) }136 if text.count > 512 { diagnostics.append(Diagnostic(code: "E-LIMIT", "String longer than 512 characters.", at: loc, length: i - start)) }137 tokens.append(make(.string(text), start, loc, i)); continue138 }139 // Symbols, longest first.140 let three = String(chars[i ..< min(i + 3, chars.count)])141 let two = String(three.prefix(2))142 if ["...", "..<"].contains(three) { advance(); advance(); advance(); tokens.append(make(.symbol(three), start, loc, i)); continue }143 if ["->", "==", "!=", "<=", ">=", "&&", "||"].contains(two) { advance(); advance(); tokens.append(make(.symbol(two), start, loc, i)); continue }144 if "{}()[],.:=<>+-*/%!".contains(c) { advance(); tokens.append(make(.symbol(String(c)), start, loc, i)); continue }145 diagnostics.append(Diagnostic(code: "E-LEX", "Unexpected character '\(c)'.", at: loc))146 advance()147 }148 tokens.append(Token(kind: .eof, location: SourceLocation(line: line, column: column), length: 0, utf16Range: utf16Offsets[chars.count] ..< utf16Offsets[chars.count]))149 return mergeWordUnits(tokens)150 }151152 private mutating func scanDigits() {153 while Self.isDigit(peek()) || (peek() == "_" && Self.isDigit(peek(1))) { advance() }154 }155156 private mutating func lexNumber(_ start: Int, _ loc: SourceLocation, _ make: (TokenKind, Int, SourceLocation, Int) -> Token) -> Token {157 scanDigits()158 var isInteger = true159 if peek() == ".", Self.isDigit(peek(1)) { isInteger = false; advance(); scanDigits() }160 var value = Double(String(chars[start ..< i]).replacingOccurrences(of: "_", with: "")) ?? 0161 // Fraction with a musical unit: 1/4beat, 3/2bars.162 if isInteger, peek() == "/", Self.isDigit(peek(1)) {163 var j = i + 1164 while j < chars.count, Self.isDigit(chars[j]) { j += 1 }165 // The unit may follow directly (1/4beat) or after one space (1/4 beat).166 var u = j167 if u < chars.count, chars[u] == " " { u += 1 }168 var k = u169 while k < chars.count, Self.isIdentPart(chars[k]) { k += 1 }170 let unit = String(chars[u ..< k])171 if Self.wordUnits.contains(unit), let den = Double(String(chars[i + 1 ..< j])), den != 0 {172 while i < k { advance() }173 return make(.literal(Self.unitValue(value / den, unit)!), start, loc, i)174 }175 }176 // Adjacent unit.177 if let c = peek(), Self.isIdentStart(c) {178 var k = i179 while k < chars.count, Self.isIdentPart(chars[k]) { k += 1 }180 let unit = String(chars[i ..< k])181 if let v = Self.unitValue(value, unit), Self.units.contains(unit) {182 while i < k { advance() }183 return make(.literal(v), start, loc, i)184 }185 while i < k { advance() }186 diagnostics.append(Diagnostic(code: "E-UNIT", "Unknown unit '\(unit)'. Units: Hz, kHz, dB, beat(s), bar(s), s, ms, min, bpm, st, %.", at: loc, length: i - start))187 return make(.number(value), start, loc, i)188 }189 // Percent: adjacent '%' that is not a modulo operator.190 if peek() == "%" {191 let next: Character? = i + 1 < chars.count ? chars[i + 1] : nil192 let isModulo = next.map { Self.isDigit($0) || Self.isIdentStart($0) || $0 == "(" } ?? false193 if !isModulo { advance(); value /= 100; return make(.literal(.number(value)), start, loc, i) }194 }195 return make(.number(value), start, loc, i)196 }197198 /// `8 bars` (number, space, word unit) becomes one literal, as in `over 8 bars`.199 private func mergeWordUnits(_ tokens: [Token]) -> [Token] {200 var out: [Token] = []; out.reserveCapacity(tokens.count)201 var k = 0202 while k < tokens.count {203 let t = tokens[k]204 if case let .number(x) = t.kind, k + 1 < tokens.count, case let .identifier(name) = tokens[k + 1].kind, Self.wordUnits.contains(name),205 tokens[k + 1].location.line == t.location.line {206 let n = tokens[k + 1]207 out.append(Token(kind: .literal(Self.unitValue(x, name)!), location: t.location, length: n.location.column + n.length - t.location.column,208 utf16Range: t.utf16Range.lowerBound ..< n.utf16Range.upperBound))209 k += 2; continue210 }211 out.append(t); k += 1212 }213 return out214 }215216 /// Syntax colouring spans for arbitrary (possibly invalid) source.217 public static func highlight(_ source: String) -> [HighlightSpan] {218 var lexer = Lexer(source)219 let tokens = lexer.tokenize()220 var spans: [HighlightSpan] = lexer.comments.map { HighlightSpan(range: $0, tokenClass: .comment) }221 let keywords = Set(ScriptSchema.keywords)222 let roots: Set<String> = ["deck", "mixer", "fx", "master"]223 for t in tokens {224 switch t.kind {225 case let .identifier(name):226 spans.append(HighlightSpan(range: t.utf16Range, tokenClass: roots.contains(name) ? .root : (keywords.contains(name) ? .keyword : .identifier)))227 case .number, .literal: spans.append(HighlightSpan(range: t.utf16Range, tokenClass: .number))228 case .string: spans.append(HighlightSpan(range: t.utf16Range, tokenClass: .string))229 case .symbol: spans.append(HighlightSpan(range: t.utf16Range, tokenClass: .symbol))230 default: break231 }232 }233 return spans.sorted { $0.range.lowerBound < $1.range.lowerBound }234 }235}236