Repository Public License
1// ORIX OS Automation2// File: Packages/ORIXFeatureCode/Tests/ORIXFeatureCodeTests/TokenizerTests.swift3// Description: Unit tests of the syntax tokenizers: every language, strings, comments, heredocs, block scalars and workflow expressions.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import Testing9@testable import ORIXFeatureCode1011/// Token texts with their kinds, in order.12func lex(_ text: String, _ language: CodeLanguage) -> [(String, TokenKind)] {13 let ns = text as NSString14 return SyntaxTokenizer.tokenize(text, language: language).map { (ns.substring(with: $0.range), $0.kind) }15}1617func has(_ tokens: [(String, TokenKind)], _ text: String, _ kind: TokenKind) -> Bool {18 tokens.contains { $0.0 == text && $0.1 == kind }19}2021@Suite("Unit · tokenizers") struct TokenizerTests {22 @Test func yamlKeysScalarsAndComments() {23 let t = lex("# top\nname: \"Weekly: report\" # trailing\ncount: 42\nratio: -0.5e3\nflag: true\nplain: some text # note\nzero: 007\n", .yaml)24 #expect(has(t, "# top", .comment))25 #expect(has(t, "name", .key))26 #expect(has(t, "\"Weekly: report\"", .string))27 #expect(has(t, "# trailing", .comment))28 #expect(has(t, "42", .number))29 #expect(has(t, "-0.5e3", .number))30 #expect(has(t, "true", .constant))31 #expect(has(t, "# note", .comment))32 #expect(!has(t, "007", .number)) // leading zeros are strings33 #expect(!t.contains { $0.0.contains("some text") && $0.1 == .string })34 }3536 @Test func yamlHashInsideAWordIsNotAComment() {37 let t = lex("url: http://example.com/#anchor\ncolor: a#b\n", .yaml)38 #expect(!t.contains { $0.1 == .comment })39 }4041 @Test func yamlExpressionsInsidePlainAndQuotedScalars() {42 let t = lex("path: ${{ params.folder }}/out-${{ env.DATE }}.md\nq: \"${{ len(steps.fetch-data.output.items) > 0 }}\"\nlit: $${{ not }}\n", .yaml)43 #expect(t.filter { $0.0 == "${{" && $0.1 == .interpolation }.count == 3)44 #expect(t.filter { $0.0 == "}}" && $0.1 == .interpolation }.count == 3)45 #expect(has(t, "params", .variable))46 #expect(has(t, "env", .variable))47 #expect(has(t, "len", .function))48 #expect(has(t, "steps", .variable))49 #expect(has(t, "0", .number))50 #expect(has(t, ">", .op))51 #expect(!has(t, "not", .variable))52 }5354 @Test func yamlAColonInsideAnExpressionDoesNotMakeAKey() {55 let t = lex("when: ${{ contains(vars.map, \"a: b\") }}\n", .yaml)56 #expect(t.filter { $0.1 == .key }.map(\.0) == ["when"])57 #expect(has(t, "\"a: b\"", .string))58 }5960 @Test func yamlBlockScalarsSpanLinesAndEnd() {61 let source = "steps:\n - id: run\n with:\n script: |\n echo \"hi\" # not a comment\n key: value\n\n last line\n cwd: /tmp\n"62 let t = lex(source, .yaml)63 #expect(has(t, "|", .punctuation))64 #expect(has(t, "echo \"hi\" # not a comment", .string))65 #expect(has(t, "key: value", .string))66 #expect(has(t, "last line", .string))67 #expect(has(t, "cwd", .key))68 #expect(!has(t, "key", .key))69 // A block scalar that is a list item.70 let item = lex("items:\n - >-\n folded text\n - next\n", .yaml)71 #expect(has(item, "folded text", .string))72 #expect(!item.contains { $0.0.contains("next") && $0.1 == .string })73 }7475 @Test func yamlFlowCollectionsAndToolIDs() {76 let t = lex("needs: [a, b]\nwith: {path: \"~/x\", n: 3}\ntool: filesystem.list\n- tool: shell.run\n", .yaml)77 #expect(t.filter { $0.0 == "[" || $0.0 == "]" || $0.0 == "{" || $0.0 == "}" }.allSatisfy { $0.1 == .punctuation })78 #expect(has(t, "path", .key))79 #expect(has(t, "3", .number))80 #expect(has(t, "filesystem.list", .function))81 #expect(has(t, "shell.run", .function))82 }8384 @Test func jsonKeysStringsNumbersAndExpressions() {85 let t = lex("{\n \"name\": \"a \\\"quoted\\\" b\",\n \"n\": -12.5e2, \"ok\": true, \"none\": null,\n \"path\": \"${{ params.x }}/y\"\n}\n", .json)86 #expect(has(t, "\"name\"", .key))87 #expect(has(t, "\"a \\\"quoted\\\" b\"", .string))88 #expect(has(t, "-12.5e2", .number))89 #expect(has(t, "true", .constant))90 #expect(has(t, "null", .constant))91 #expect(has(t, "params", .variable))92 #expect(has(t, "\"", .string))93 #expect(has(t, "/y\"", .string))94 }9596 @Test func pythonStringsCommentsAndDefinitions() {97 let source = "import os # comment with 'quote'\n@decorator\ndef greet(name: str) -> str:\n \"\"\"Doc string\n spans # lines\n \"\"\"\n x = r'raw\\n' + f\"{name!r}\" + b'bytes'\n return 0x1F + 1_000 + 3.14e-2 if name is None else len(name)\nclass Box:\n pass\ns = 'it''s # not a comment'\n"98 let t = lex(source, .python)99 #expect(has(t, "# comment with 'quote'", .comment))100 #expect(has(t, "@decorator", .attribute))101 #expect(has(t, "def", .keyword))102 #expect(has(t, "greet", .function))103 #expect(has(t, "\"\"\"Doc string", .string))104 #expect(has(t, " spans # lines", .string))105 #expect(has(t, " \"\"\"", .string))106 #expect(has(t, "r'raw\\n'", .string))107 #expect(has(t, "f\"{name!r}\"", .string))108 #expect(has(t, "b'bytes'", .string))109 #expect(has(t, "0x1F", .number))110 #expect(has(t, "1_000", .number))111 #expect(has(t, "3.14e-2", .number))112 #expect(has(t, "None", .constant))113 #expect(has(t, "len", .function))114 #expect(has(t, "Box", .type))115 #expect(!t.contains { $0.0.contains("not a comment") && $0.1 == .comment })116 }117118 @Test func bashQuotesVariablesAndHeredocs() {119 let source = "#!/bin/bash\nname=\"World $USER ${HOME}\" # greet\necho 'single $notvar # no comment'\necho $# ${#arr[@]} a#b\ncat <<EOF > out.txt\nline with $var and # hash\nEOF\ncat <<-'END'\n\tindented body\n\tEND\nif [ -f x ]; then echo ok; fi\ngrep <<< \"here string\"\n"120 let t = lex(source, .bash)121 #expect(has(t, "#!/bin/bash", .comment))122 #expect(has(t, "$USER", .variable))123 #expect(has(t, "${HOME}", .variable))124 #expect(has(t, "# greet", .comment))125 #expect(has(t, "'single $notvar # no comment'", .string))126 #expect(has(t, "$#", .variable))127 #expect(has(t, "${#arr[@]}", .variable))128 #expect(t.filter { $0.1 == .comment }.count == 2) // shebang and "# greet" only129 #expect(has(t, "line with $var and # hash", .string))130 #expect(has(t, "EOF", .constant))131 #expect(has(t, "\tindented body", .string))132 #expect(has(t, "END", .constant))133 #expect(has(t, "if", .keyword))134 #expect(has(t, "fi", .keyword))135 #expect(has(t, "\"here string\"", .string))136 }137138 @Test func bashStringsSpanLines() {139 let t = lex("msg=\"first\nsecond $x\nthird\"\necho done\n", .bash)140 #expect(has(t, "\"first", .string))141 #expect(has(t, "$x", .variable))142 #expect(has(t, "third\"", .string))143 #expect(has(t, "echo", .function))144 #expect(has(t, "done", .keyword))145 }146147 @Test func appleScriptCommentsAndKeywords() {148 let source = "-- line comment\n(* block (* nested *) still\n comment *) tell application \"Finder\"\n set n to \"a \\\"b\\\" -- c\"\n Display Dialog n\nEND TELL\n# hash comment\n"149 let t = lex(source, .applescript)150 #expect(has(t, "-- line comment", .comment))151 #expect(has(t, "(* block (* nested *) still", .comment))152 #expect(has(t, " comment *)", .comment))153 #expect(has(t, "tell", .keyword))154 #expect(has(t, "\"Finder\"", .string))155 #expect(has(t, "\"a \\\"b\\\" -- c\"", .string))156 #expect(has(t, "Display", .function)) // case-insensitive157 #expect(has(t, "END", .keyword))158 #expect(has(t, "# hash comment", .comment))159 }160161 @Test func swiftNestedCommentsInterpolationAndMultilineStrings() {162 let source = "/* outer /* inner */ still */ let a = 1\n// line\n@MainActor func run(_ x: Int) -> String {\n let s = \"value \\(x + count(\"q\")) end\" // after\n let m = \"\"\"\n multi \"quoted\" // not comment\n \"\"\"\n #if DEBUG\n return s + m\n #endif\n}\nstruct Box<T> {}\n"163 let t = lex(source, .swift)164 #expect(has(t, "/* outer /* inner */ still */", .comment))165 #expect(has(t, "let", .keyword))166 #expect(has(t, "// line", .comment))167 #expect(has(t, "@MainActor", .attribute))168 #expect(has(t, "run", .function))169 #expect(has(t, "Int", .type))170 #expect(has(t, "\"value \\(x + count(\"q\")) end\"", .string))171 #expect(has(t, "// after", .comment))172 #expect(has(t, " multi \"quoted\" // not comment", .string))173 #expect(has(t, "#if", .keyword))174 #expect(has(t, "Box", .type))175 }176177 @Test func javaScriptTemplatesRegexAndComments() {178 let source = "const re = /ab+c[/]\\/x/gi; // trailing\nconst t = `line ${a}\nsecond // not comment`;\n/* block\n comment */ let half = a / b / c;\nfunction go() { return 'it\\'s'; }\n"179 let t = lex(source, .javascript)180 #expect(has(t, "/ab+c[/]\\/x/gi", .string))181 #expect(has(t, "// trailing", .comment))182 #expect(has(t, "`line ${a}", .string))183 #expect(has(t, "second // not comment`", .string))184 #expect(has(t, "/* block", .comment))185 #expect(has(t, " comment */", .comment))186 #expect(!t.contains { $0.0.hasPrefix("/ b") }) // division is not a regex187 #expect(has(t, "go", .function))188 #expect(has(t, "'it\\'s'", .string))189 }190191 @Test func tokensNeverOverlapAndStayInBounds() {192 let samples: [(String, CodeLanguage)] = [193 (WorkflowSnippets.starter, .yaml), ("{\"a\": [1, \"${{ x }}\"]}", .json),194 (ScriptTemplates.templates(.python).map(\.body).joined(), .python), (ScriptTemplates.templates(.bash).map(\.body).joined(), .bash),195 (ScriptTemplates.templates(.applescript).map(\.body).joined(), .applescript), (ScriptTemplates.templates(.swift).map(\.body).joined(), .swift),196 (ScriptTemplates.templates(.javascript).map(\.body).joined() + ScriptTemplates.templates(.jxa).map(\.body).joined(), .javascript),197 ("\"unterminated\n'also\n${{ open\n/* never closed", .yaml), ("\"\"\"open\n<<EOF\n`tick", .bash),198 ]199 for (text, language) in samples {200 let tokens = SyntaxTokenizer.tokenize(text, language: language)201 var end = 0202 for t in tokens {203 #expect(t.range.location >= end, "overlap in \(language)")204 #expect(t.range.length > 0)205 end = NSMaxRange(t.range)206 }207 #expect(end <= (text as NSString).length)208 }209 }210}211