SPB Git

Repository Public License

TokenizerTests.swift 10.7 KB 211 lines swift

1// ORIX OS Automation2// File: Packages/ORIXFeatureCode/Tests/ORIXFeatureCodeTests/TokenizerTests.swift3// Description: Unit tests of the syntax tokenizers: every language, strings, comments, heredocs, block scalars and workflow expressions.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9@testable import ORIXFeatureCode1011/// Token texts with their kinds, in order.12func lex(_ text: String, _ language: CodeLanguage) -> [(String, TokenKind)] {13    let ns = text as NSString14    return SyntaxTokenizer.tokenize(text, language: language).map { (ns.substring(with: $0.range), $0.kind) }15}1617func has(_ tokens: [(String, TokenKind)], _ text: String, _ kind: TokenKind) -> Bool {18    tokens.contains { $0.0 == text && $0.1 == kind }19}2021@Suite("Unit · tokenizers") struct TokenizerTests {22    @Test func yamlKeysScalarsAndComments() {23        let t = lex("# top\nname: \"Weekly: report\"  # trailing\ncount: 42\nratio: -0.5e3\nflag: true\nplain: some text # note\nzero: 007\n", .yaml)24        #expect(has(t, "# top", .comment))25        #expect(has(t, "name", .key))26        #expect(has(t, "\"Weekly: report\"", .string))27        #expect(has(t, "# trailing", .comment))28        #expect(has(t, "42", .number))29        #expect(has(t, "-0.5e3", .number))30        #expect(has(t, "true", .constant))31        #expect(has(t, "# note", .comment))32        #expect(!has(t, "007", .number))                       // leading zeros are strings33        #expect(!t.contains { $0.0.contains("some text") && $0.1 == .string })34    }3536    @Test func yamlHashInsideAWordIsNotAComment() {37        let t = lex("url: http://example.com/#anchor\ncolor: a#b\n", .yaml)38        #expect(!t.contains { $0.1 == .comment })39    }4041    @Test func yamlExpressionsInsidePlainAndQuotedScalars() {42        let t = lex("path: ${{ params.folder }}/out-${{ env.DATE }}.md\nq: \"${{ len(steps.fetch-data.output.items) > 0 }}\"\nlit: $${{ not }}\n", .yaml)43        #expect(t.filter { $0.0 == "${{" && $0.1 == .interpolation }.count == 3)44        #expect(t.filter { $0.0 == "}}" && $0.1 == .interpolation }.count == 3)45        #expect(has(t, "params", .variable))46        #expect(has(t, "env", .variable))47        #expect(has(t, "len", .function))48        #expect(has(t, "steps", .variable))49        #expect(has(t, "0", .number))50        #expect(has(t, ">", .op))51        #expect(!has(t, "not", .variable))52    }5354    @Test func yamlAColonInsideAnExpressionDoesNotMakeAKey() {55        let t = lex("when: ${{ contains(vars.map, \"a: b\") }}\n", .yaml)56        #expect(t.filter { $0.1 == .key }.map(\.0) == ["when"])57        #expect(has(t, "\"a: b\"", .string))58    }5960    @Test func yamlBlockScalarsSpanLinesAndEnd() {61        let source = "steps:\n  - id: run\n    with:\n      script: |\n        echo \"hi\"   # not a comment\n        key: value\n\n        last line\n      cwd: /tmp\n"62        let t = lex(source, .yaml)63        #expect(has(t, "|", .punctuation))64        #expect(has(t, "echo \"hi\"   # not a comment", .string))65        #expect(has(t, "key: value", .string))66        #expect(has(t, "last line", .string))67        #expect(has(t, "cwd", .key))68        #expect(!has(t, "key", .key))69        // A block scalar that is a list item.70        let item = lex("items:\n  - >-\n    folded text\n  - next\n", .yaml)71        #expect(has(item, "folded text", .string))72        #expect(!item.contains { $0.0.contains("next") && $0.1 == .string })73    }7475    @Test func yamlFlowCollectionsAndToolIDs() {76        let t = lex("needs: [a, b]\nwith: {path: \"~/x\", n: 3}\ntool: filesystem.list\n- tool: shell.run\n", .yaml)77        #expect(t.filter { $0.0 == "[" || $0.0 == "]" || $0.0 == "{" || $0.0 == "}" }.allSatisfy { $0.1 == .punctuation })78        #expect(has(t, "path", .key))79        #expect(has(t, "3", .number))80        #expect(has(t, "filesystem.list", .function))81        #expect(has(t, "shell.run", .function))82    }8384    @Test func jsonKeysStringsNumbersAndExpressions() {85        let t = lex("{\n  \"name\": \"a \\\"quoted\\\" b\",\n  \"n\": -12.5e2, \"ok\": true, \"none\": null,\n  \"path\": \"${{ params.x }}/y\"\n}\n", .json)86        #expect(has(t, "\"name\"", .key))87        #expect(has(t, "\"a \\\"quoted\\\" b\"", .string))88        #expect(has(t, "-12.5e2", .number))89        #expect(has(t, "true", .constant))90        #expect(has(t, "null", .constant))91        #expect(has(t, "params", .variable))92        #expect(has(t, "\"", .string))93        #expect(has(t, "/y\"", .string))94    }9596    @Test func pythonStringsCommentsAndDefinitions() {97        let source = "import os  # comment with 'quote'\n@decorator\ndef greet(name: str) -> str:\n    \"\"\"Doc string\n    spans # lines\n    \"\"\"\n    x = r'raw\\n' + f\"{name!r}\" + b'bytes'\n    return 0x1F + 1_000 + 3.14e-2 if name is None else len(name)\nclass Box:\n    pass\ns = 'it''s # not a comment'\n"98        let t = lex(source, .python)99        #expect(has(t, "# comment with 'quote'", .comment))100        #expect(has(t, "@decorator", .attribute))101        #expect(has(t, "def", .keyword))102        #expect(has(t, "greet", .function))103        #expect(has(t, "\"\"\"Doc string", .string))104        #expect(has(t, "    spans # lines", .string))105        #expect(has(t, "    \"\"\"", .string))106        #expect(has(t, "r'raw\\n'", .string))107        #expect(has(t, "f\"{name!r}\"", .string))108        #expect(has(t, "b'bytes'", .string))109        #expect(has(t, "0x1F", .number))110        #expect(has(t, "1_000", .number))111        #expect(has(t, "3.14e-2", .number))112        #expect(has(t, "None", .constant))113        #expect(has(t, "len", .function))114        #expect(has(t, "Box", .type))115        #expect(!t.contains { $0.0.contains("not a comment") && $0.1 == .comment })116    }117118    @Test func bashQuotesVariablesAndHeredocs() {119        let source = "#!/bin/bash\nname=\"World $USER ${HOME}\" # greet\necho 'single $notvar # no comment'\necho $# ${#arr[@]} a#b\ncat <<EOF > out.txt\nline with $var and # hash\nEOF\ncat <<-'END'\n\tindented body\n\tEND\nif [ -f x ]; then echo ok; fi\ngrep <<< \"here string\"\n"120        let t = lex(source, .bash)121        #expect(has(t, "#!/bin/bash", .comment))122        #expect(has(t, "$USER", .variable))123        #expect(has(t, "${HOME}", .variable))124        #expect(has(t, "# greet", .comment))125        #expect(has(t, "'single $notvar # no comment'", .string))126        #expect(has(t, "$#", .variable))127        #expect(has(t, "${#arr[@]}", .variable))128        #expect(t.filter { $0.1 == .comment }.count == 2)       // shebang and "# greet" only129        #expect(has(t, "line with $var and # hash", .string))130        #expect(has(t, "EOF", .constant))131        #expect(has(t, "\tindented body", .string))132        #expect(has(t, "END", .constant))133        #expect(has(t, "if", .keyword))134        #expect(has(t, "fi", .keyword))135        #expect(has(t, "\"here string\"", .string))136    }137138    @Test func bashStringsSpanLines() {139        let t = lex("msg=\"first\nsecond $x\nthird\"\necho done\n", .bash)140        #expect(has(t, "\"first", .string))141        #expect(has(t, "$x", .variable))142        #expect(has(t, "third\"", .string))143        #expect(has(t, "echo", .function))144        #expect(has(t, "done", .keyword))145    }146147    @Test func appleScriptCommentsAndKeywords() {148        let source = "-- line comment\n(* block (* nested *) still\n   comment *) tell application \"Finder\"\n    set n to \"a \\\"b\\\" -- c\"\n    Display Dialog n\nEND TELL\n# hash comment\n"149        let t = lex(source, .applescript)150        #expect(has(t, "-- line comment", .comment))151        #expect(has(t, "(* block (* nested *) still", .comment))152        #expect(has(t, "   comment *)", .comment))153        #expect(has(t, "tell", .keyword))154        #expect(has(t, "\"Finder\"", .string))155        #expect(has(t, "\"a \\\"b\\\" -- c\"", .string))156        #expect(has(t, "Display", .function))                    // case-insensitive157        #expect(has(t, "END", .keyword))158        #expect(has(t, "# hash comment", .comment))159    }160161    @Test func swiftNestedCommentsInterpolationAndMultilineStrings() {162        let source = "/* outer /* inner */ still */ let a = 1\n// line\n@MainActor func run(_ x: Int) -> String {\n    let s = \"value \\(x + count(\"q\")) end\" // after\n    let m = \"\"\"\n    multi \"quoted\" // not comment\n    \"\"\"\n    #if DEBUG\n    return s + m\n    #endif\n}\nstruct Box<T> {}\n"163        let t = lex(source, .swift)164        #expect(has(t, "/* outer /* inner */ still */", .comment))165        #expect(has(t, "let", .keyword))166        #expect(has(t, "// line", .comment))167        #expect(has(t, "@MainActor", .attribute))168        #expect(has(t, "run", .function))169        #expect(has(t, "Int", .type))170        #expect(has(t, "\"value \\(x + count(\"q\")) end\"", .string))171        #expect(has(t, "// after", .comment))172        #expect(has(t, "    multi \"quoted\" // not comment", .string))173        #expect(has(t, "#if", .keyword))174        #expect(has(t, "Box", .type))175    }176177    @Test func javaScriptTemplatesRegexAndComments() {178        let source = "const re = /ab+c[/]\\/x/gi; // trailing\nconst t = `line ${a}\nsecond // not comment`;\n/* block\n   comment */ let half = a / b / c;\nfunction go() { return 'it\\'s'; }\n"179        let t = lex(source, .javascript)180        #expect(has(t, "/ab+c[/]\\/x/gi", .string))181        #expect(has(t, "// trailing", .comment))182        #expect(has(t, "`line ${a}", .string))183        #expect(has(t, "second // not comment`", .string))184        #expect(has(t, "/* block", .comment))185        #expect(has(t, "   comment */", .comment))186        #expect(!t.contains { $0.0.hasPrefix("/ b") })            // division is not a regex187        #expect(has(t, "go", .function))188        #expect(has(t, "'it\\'s'", .string))189    }190191    @Test func tokensNeverOverlapAndStayInBounds() {192        let samples: [(String, CodeLanguage)] = [193            (WorkflowSnippets.starter, .yaml), ("{\"a\": [1, \"${{ x }}\"]}", .json),194            (ScriptTemplates.templates(.python).map(\.body).joined(), .python), (ScriptTemplates.templates(.bash).map(\.body).joined(), .bash),195            (ScriptTemplates.templates(.applescript).map(\.body).joined(), .applescript), (ScriptTemplates.templates(.swift).map(\.body).joined(), .swift),196            (ScriptTemplates.templates(.javascript).map(\.body).joined() + ScriptTemplates.templates(.jxa).map(\.body).joined(), .javascript),197            ("\"unterminated\n'also\n${{ open\n/* never closed", .yaml), ("\"\"\"open\n<<EOF\n`tick", .bash),198        ]199        for (text, language) in samples {200            let tokens = SyntaxTokenizer.tokenize(text, language: language)201            var end = 0202            for t in tokens {203                #expect(t.range.location >= end, "overlap in \(language)")204                #expect(t.range.length > 0)205                end = NSMaxRange(t.range)206            }207            #expect(end <= (text as NSString).length)208        }209    }210}211
Files main
Menu

Theme