SPB Git

Repository Public

Lexer.swift 11.1 KB 236 lines swift

1// ORIX DJ2// File: Packages/ORIXDJCore/Sources/ORIXScript/Lexer.swift3// Description: ORIX Script lexer: source text to tokens, including unit literals (-12dB, 20Hz, 4bars, 1/4beat, 50%).4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation89public enum TokenKind: Equatable, Sendable {10    case identifier(String)11    case number(Double)12    /// A number with a unit, already converted to a value.13    case literal(Value)14    case string(String)15    case symbol(String)16    case newline17    case eof18}1920public struct Token: Equatable, Sendable {21    public var kind: TokenKind22    public var location: SourceLocation23    public var length: Int24    /// Offset range in the source (UTF-16 units), for the editor's syntax colouring.25    public var utf16Range: Range<Int>26}2728public enum TokenClass: Sendable { case keyword, identifier, number, string, comment, symbol, root }2930/// A coloured span for the editor. Produced even for text that does not parse.31public struct HighlightSpan: Sendable, Equatable {32    public var range: Range<Int>33    public var tokenClass: TokenClass34}3536public struct Lexer {37    private let chars: [Character]38    private let utf16Offsets: [Int]39    private var i = 040    private var line = 141    private var column = 142    public private(set) var diagnostics: [Diagnostic] = []43    public private(set) var comments: [Range<Int>] = []4445    public init(_ source: String) {46        chars = Array(source)47        var offsets: [Int] = []; offsets.reserveCapacity(chars.count + 1)48        var o = 049        for c in chars { offsets.append(o); o += c.utf16.count }50        offsets.append(o)51        utf16Offsets = offsets52    }5354    static let units: Set<String> = ["Hz", "kHz", "dB", "beat", "beats", "bar", "bars", "s", "sec", "ms", "min", "bpm", "st"]55    static let wordUnits: Set<String> = ["beat", "beats", "bar", "bars"]5657    static func unitValue(_ x: Double, _ unit: String) -> Value? {58        switch unit {59        case "Hz": .frequency(x)60        case "kHz": .frequency(x * 1000)61        case "dB": .gain(x)62        case "beat", "beats": .duration(.beats(x))63        case "bar", "bars": .duration(.beats(x * ORIXScriptLanguage.beatsPerBar))64        case "s", "sec": .duration(.seconds(x))65        case "ms": .duration(.seconds(x / 1000))66        case "min": .duration(.seconds(x * 60))67        case "bpm", "st": .number(x)68        case "%": .number(x / 100)69        default: nil70        }71    }7273    private func peek(_ k: Int = 0) -> Character? { i + k < chars.count ? chars[i + k] : nil }7475    private mutating func advance() {76        if chars[i] == "\n" { line += 1; column = 1 } else { column += 1 }77        i += 178    }7980    private static func isIdentStart(_ c: Character) -> Bool { c.isLetter || c == "_" }81    private static func isIdentPart(_ c: Character) -> Bool { c.isLetter || c.isNumber || c == "_" }82    private static func isDigit(_ c: Character?) -> Bool { c.map { $0.isASCII && $0.isNumber } ?? false }8384    public mutating func tokenize() -> [Token] {85        var tokens: [Token] = []86        let offsets = utf16Offsets87        let make: (TokenKind, Int, SourceLocation, Int) -> Token = { kind, start, loc, end in88            Token(kind: kind, location: loc, length: end - start, utf16Range: offsets[start] ..< offsets[end])89        }90        while let c = peek() {91            let start = i; let loc = SourceLocation(line: line, column: column)92            if c == "\n" { advance(); tokens.append(make(.newline, start, loc, i)); continue }93            if c == ";" { advance(); tokens.append(make(.newline, start, loc, i)); continue }94            if c.isWhitespace { advance(); continue }95            if c == "/", peek(1) == "/" {96                while let d = peek(), d != "\n" { _ = d; advance() }97                comments.append(utf16Offsets[start] ..< utf16Offsets[i]); continue98            }99            if c == "/", peek(1) == "*" {100                advance(); advance()101                var closed = false102                while peek() != nil {103                    if peek() == "*", peek(1) == "/" { advance(); advance(); closed = true; break }104                    advance()105                }106                if !closed { diagnostics.append(Diagnostic(code: "E-LEX", "Unterminated block comment.", at: loc, length: 2)) }107                comments.append(utf16Offsets[start] ..< utf16Offsets[i]); continue108            }109            if Self.isDigit(c) || (c == "." && Self.isDigit(peek(1))) {110                tokens.append(lexNumber(start, loc, make)); continue111            }112            if Self.isIdentStart(c) {113                while let d = peek(), Self.isIdentPart(d) { advance() }114                tokens.append(make(.identifier(String(chars[start ..< i])), start, loc, i)); continue115            }116            if c == "\"" {117                advance()118                var text = ""; var closed = false119                while let d = peek() {120                    if d == "\n" { break }121                    if d == "\\" {122                        advance()123                        switch peek() {124                        case "n": text.append("\n")125                        case "t": text.append("\t")126                        case let e?: text.append(e)127                        case nil: break128                        }129                        if peek() != nil { advance() }130                        continue131                    }132                    if d == "\"" { advance(); closed = true; break }133                    text.append(d); advance()134                }135                if !closed { diagnostics.append(Diagnostic(code: "E-LEX", "Unterminated string.", at: loc, length: i - start)) }136                if text.count > 512 { diagnostics.append(Diagnostic(code: "E-LIMIT", "String longer than 512 characters.", at: loc, length: i - start)) }137                tokens.append(make(.string(text), start, loc, i)); continue138            }139            // Symbols, longest first.140            let three = String(chars[i ..< min(i + 3, chars.count)])141            let two = String(three.prefix(2))142            if ["...", "..<"].contains(three) { advance(); advance(); advance(); tokens.append(make(.symbol(three), start, loc, i)); continue }143            if ["->", "==", "!=", "<=", ">=", "&&", "||"].contains(two) { advance(); advance(); tokens.append(make(.symbol(two), start, loc, i)); continue }144            if "{}()[],.:=<>+-*/%!".contains(c) { advance(); tokens.append(make(.symbol(String(c)), start, loc, i)); continue }145            diagnostics.append(Diagnostic(code: "E-LEX", "Unexpected character '\(c)'.", at: loc))146            advance()147        }148        tokens.append(Token(kind: .eof, location: SourceLocation(line: line, column: column), length: 0, utf16Range: utf16Offsets[chars.count] ..< utf16Offsets[chars.count]))149        return mergeWordUnits(tokens)150    }151152    private mutating func scanDigits() {153        while Self.isDigit(peek()) || (peek() == "_" && Self.isDigit(peek(1))) { advance() }154    }155156    private mutating func lexNumber(_ start: Int, _ loc: SourceLocation, _ make: (TokenKind, Int, SourceLocation, Int) -> Token) -> Token {157        scanDigits()158        var isInteger = true159        if peek() == ".", Self.isDigit(peek(1)) { isInteger = false; advance(); scanDigits() }160        var value = Double(String(chars[start ..< i]).replacingOccurrences(of: "_", with: "")) ?? 0161        // Fraction with a musical unit: 1/4beat, 3/2bars.162        if isInteger, peek() == "/", Self.isDigit(peek(1)) {163            var j = i + 1164            while j < chars.count, Self.isDigit(chars[j]) { j += 1 }165            // The unit may follow directly (1/4beat) or after one space (1/4 beat).166            var u = j167            if u < chars.count, chars[u] == " " { u += 1 }168            var k = u169            while k < chars.count, Self.isIdentPart(chars[k]) { k += 1 }170            let unit = String(chars[u ..< k])171            if Self.wordUnits.contains(unit), let den = Double(String(chars[i + 1 ..< j])), den != 0 {172                while i < k { advance() }173                return make(.literal(Self.unitValue(value / den, unit)!), start, loc, i)174            }175        }176        // Adjacent unit.177        if let c = peek(), Self.isIdentStart(c) {178            var k = i179            while k < chars.count, Self.isIdentPart(chars[k]) { k += 1 }180            let unit = String(chars[i ..< k])181            if let v = Self.unitValue(value, unit), Self.units.contains(unit) {182                while i < k { advance() }183                return make(.literal(v), start, loc, i)184            }185            while i < k { advance() }186            diagnostics.append(Diagnostic(code: "E-UNIT", "Unknown unit '\(unit)'. Units: Hz, kHz, dB, beat(s), bar(s), s, ms, min, bpm, st, %.", at: loc, length: i - start))187            return make(.number(value), start, loc, i)188        }189        // Percent: adjacent '%' that is not a modulo operator.190        if peek() == "%" {191            let next: Character? = i + 1 < chars.count ? chars[i + 1] : nil192            let isModulo = next.map { Self.isDigit($0) || Self.isIdentStart($0) || $0 == "(" } ?? false193            if !isModulo { advance(); value /= 100; return make(.literal(.number(value)), start, loc, i) }194        }195        return make(.number(value), start, loc, i)196    }197198    /// `8 bars` (number, space, word unit) becomes one literal, as in `over 8 bars`.199    private func mergeWordUnits(_ tokens: [Token]) -> [Token] {200        var out: [Token] = []; out.reserveCapacity(tokens.count)201        var k = 0202        while k < tokens.count {203            let t = tokens[k]204            if case let .number(x) = t.kind, k + 1 < tokens.count, case let .identifier(name) = tokens[k + 1].kind, Self.wordUnits.contains(name),205               tokens[k + 1].location.line == t.location.line {206                let n = tokens[k + 1]207                out.append(Token(kind: .literal(Self.unitValue(x, name)!), location: t.location, length: n.location.column + n.length - t.location.column,208                                 utf16Range: t.utf16Range.lowerBound ..< n.utf16Range.upperBound))209                k += 2; continue210            }211            out.append(t); k += 1212        }213        return out214    }215216    /// Syntax colouring spans for arbitrary (possibly invalid) source.217    public static func highlight(_ source: String) -> [HighlightSpan] {218        var lexer = Lexer(source)219        let tokens = lexer.tokenize()220        var spans: [HighlightSpan] = lexer.comments.map { HighlightSpan(range: $0, tokenClass: .comment) }221        let keywords = Set(ScriptSchema.keywords)222        let roots: Set<String> = ["deck", "mixer", "fx", "master"]223        for t in tokens {224            switch t.kind {225            case let .identifier(name):226                spans.append(HighlightSpan(range: t.utf16Range, tokenClass: roots.contains(name) ? .root : (keywords.contains(name) ? .keyword : .identifier)))227            case .number, .literal: spans.append(HighlightSpan(range: t.utf16Range, tokenClass: .number))228            case .string: spans.append(HighlightSpan(range: t.utf16Range, tokenClass: .string))229            case .symbol: spans.append(HighlightSpan(range: t.utf16Range, tokenClass: .symbol))230            default: break231            }232        }233        return spans.sorted { $0.range.lowerBound < $1.range.lowerBound }234    }235}236
Files main
Menu

Theme