spb/metrika Public

1//2//  Lexer.swift3//  Metrika4//5//  Author:  Simon-Pierre Boucher6//  Contact: contact@spboucher.ai7//  Copyright © 2026 Simon-Pierre Boucher. All rights reserved.8//910/// Tokenizer for a single ZQL command line.11///12/// Numbers support decimal and exponent notation (`1.5e-3`). A `.` is lexed13/// as an operator when not part of a number, which lets the grammar layer14/// assemble factor-variable notation (`i.region`, `c.age#c.age`).15public struct ZQLexer: Sendable {16    public init() {}1718    public func tokenize(_ line: String) throws(ZQParseError) -> [ZQToken] {19        var tokens: [ZQToken] = []20        let chars = Array(line)21        var i = 02223        func column(_ index: Int) -> Int { index + 1 }2425        while i < chars.count {26            let c = chars[i]2728            if c.isWhitespace { i += 1; continue }2930            // Comments: // and * (only at start of line, Stata-style) end the command.31            if c == "/", i + 1 < chars.count, chars[i + 1] == "/" { break }32            if c == "*", tokens.isEmpty { break }3334            // String literal35            if c == "\"" {36                let start = i37                i += 138                var value = ""39                while i < chars.count, chars[i] != "\"" {40                    value.append(chars[i])41                    i += 142                }43                guard i < chars.count else {44                    throw ZQParseError(45                        message: "unterminated string literal",46                        column: column(start)47                    )48                }49                i += 150                tokens.append(ZQToken(kind: .string(value), column: column(start)))51                continue52            }5354            // Number: digits, or leading dot followed by a digit55            if c.isNumber || (c == "." && i + 1 < chars.count && chars[i + 1].isNumber) {56                let start = i57                var text = ""58                while i < chars.count, chars[i].isNumber || chars[i] == "." {59                    text.append(chars[i])60                    i += 161                }62                if i < chars.count, chars[i] == "e" || chars[i] == "E" {63                    var j = i + 164                    var exp = String(chars[i])65                    if j < chars.count, chars[j] == "+" || chars[j] == "-" {66                        exp.append(chars[j])67                        j += 168                    }69                    if j < chars.count, chars[j].isNumber {70                        while j < chars.count, chars[j].isNumber {71                            exp.append(chars[j])72                            j += 173                        }74                        text.append(exp)75                        i = j76                    }77                }78                guard let value = Double(text) else {79                    throw ZQParseError(80                        message: "invalid numeric literal '\(text)'",81                        column: column(start)82                    )83                }84                tokens.append(ZQToken(kind: .number(value), column: column(start)))85                continue86            }8788            // Identifier: [A-Za-z_][A-Za-z0-9_]*89            if c.isLetter || c == "_" {90                let start = i91                var text = ""92                while i < chars.count, chars[i].isLetter || chars[i].isNumber || chars[i] == "_" {93                    text.append(chars[i])94                    i += 195                }96                tokens.append(ZQToken(kind: .identifier(text), column: column(start)))97                continue98            }99100            // Punctuation & operators101            let start = i102            func push(_ kind: ZQToken.Kind, advance: Int) {103                tokens.append(ZQToken(kind: kind, column: column(start)))104                i += advance105            }106107            switch c {108            case ",": push(.comma, advance: 1)109            case ":": push(.colon, advance: 1)110            case "(": push(.lparen, advance: 1)111            case ")": push(.rparen, advance: 1)112            case "[": push(.lbracket, advance: 1)113            case "]": push(.rbracket, advance: 1)114            case "=":115                if i + 1 < chars.count, chars[i + 1] == "=" {116                    push(.op("=="), advance: 2)117                } else {118                    push(.op("="), advance: 1)119                }120            case "!":121                if i + 1 < chars.count, chars[i + 1] == "=" {122                    push(.op("!="), advance: 2)123                } else {124                    push(.op("!"), advance: 1)125                }126            case "~":127                if i + 1 < chars.count, chars[i + 1] == "=" {128                    push(.op("!="), advance: 2)129                } else {130                    throw ZQParseError(message: "unexpected character '~'", column: column(start))131                }132            case "<":133                if i + 1 < chars.count, chars[i + 1] == "=" {134                    push(.op("<="), advance: 2)135                } else {136                    push(.op("<"), advance: 1)137                }138            case ">":139                if i + 1 < chars.count, chars[i + 1] == "=" {140                    push(.op(">="), advance: 2)141                } else {142                    push(.op(">"), advance: 1)143                }144            case "&": push(.op("&"), advance: 1)145            case "|": push(.op("|"), advance: 1)146            case "+": push(.op("+"), advance: 1)147            case "-": push(.op("-"), advance: 1)148            case "*": push(.op("*"), advance: 1)149            case "/": push(.op("/"), advance: 1)150            case "^": push(.op("^"), advance: 1)151            case "#":152                if i + 1 < chars.count, chars[i + 1] == "#" {153                    push(.op("##"), advance: 2)154                } else {155                    push(.op("#"), advance: 1)156                }157            case ".": push(.op("."), advance: 1)158            default:159                throw ZQParseError(160                    message: "unexpected character '\(c)'",161                    column: column(start)162                )163            }164        }165166        tokens.append(ZQToken(kind: .endOfLine, column: chars.count + 1))167        return tokens168    }169}170