From 40e5e9989e976769a8c5a51f1100a667d8419736 Mon Sep 17 00:00:00 2001
From: Vlad Dimov <77745022+DevDimov@users.noreply.github.com>
Date: Thu, 11 Jul 2024 20:24:31 +0100
Subject: [PATCH] Add JSONata Lexer (#983)

Add lexer for [JSONata](https://docs.jsonata.org/overview.html)
including test data.
---
 lexers/embedded/jsonata.xml      |  83 ++++++++
 lexers/testdata/jsonata.actual   |  38 ++++
 lexers/testdata/jsonata.expected | 335 +++++++++++++++++++++++++++++++
 3 files changed, 456 insertions(+)
 create mode 100644 lexers/embedded/jsonata.xml
 create mode 100644 lexers/testdata/jsonata.actual
 create mode 100644 lexers/testdata/jsonata.expected

diff --git a/lexers/embedded/jsonata.xml b/lexers/embedded/jsonata.xml
new file mode 100644
index 0000000..c0eafab
--- /dev/null
+++ b/lexers/embedded/jsonata.xml
@@ -0,0 +1,83 @@
+<lexer>
+  <config>
+    <name>JSONata</name>
+    <alias>jsonata</alias>
+    <filename>*.jsonata</filename>
+    <dot_all>true</dot_all>
+  </config>
+  <rules>
+    <state name="root">
+      <rule pattern="/\*.*?\*/">
+        <token type="CommentMultiline"/>
+      </rule>
+      <rule pattern="[{}()\[\]:;,\.=]">
+        <token type="Punctuation"/>
+      </rule>
+       <rule pattern="\.\."> // Spread operator
+        <token type="Operator"/>
+      </rule> 
+      <rule pattern="\^(?=\()"> // Sort operator
+        <token type="Operator"/>
+      </rule>
+      <rule pattern="\*\*|\*(?=\.)|\*"> // Descendant | Wildcard | Multiplication
+        <token type="Operator"/>
+      </rule>
+      <rule pattern="\/(?!\*)"> // Division
+        <token type="Operator"/>
+      </rule>
+      <rule pattern="[&lt;&gt;!]=?"> // Comparison operators
+        <token type="Operator"/>
+      </rule>
+      <rule pattern="~>">
+        <token type="Operator"/>
+      </rule>
+      <rule pattern="\b(and|or|in)\b">
+        <token type="Operator"/>
+      </rule>
+      <rule pattern="[%@#&amp;?]|\+(?!\d)|\-(?!\d)">
+        <token type="Operator"/>
+      </rule>
+      <rule pattern="\$[a-zA-Z0-9_]*(?![\w\(])">
+        <token type="NameVariable"/>
+      </rule>
+      <rule pattern="\$\w*(?=\()">
+        <token type="NameFunction"/>
+      </rule>
+      <rule pattern="\s+">
+        <token type="Text"/>
+      </rule>
+      <rule pattern="(true|false)\b">
+        <token type="KeywordConstant"/>
+      </rule>
+      <rule pattern="\b(function)\b">
+        <token type="Keyword"/>
+      </rule>
+      <rule pattern="(\+|-)?(0|[1-9]\d*)(\.\d+[eE](\+|-)?\d+|[eE](\+|-)?\d+|\.\d+)">
+        <token type="LiteralNumberFloat"/>
+      </rule> 
+      <rule pattern="(\+|-)?(0|[1-9]\d*)">
+        <token type="LiteralNumberInteger"/>
+      </rule>
+      <!-- NOTE: This expression matches all object keys (NameTags), which are essentially strings with double quotes
+      that should only be captured on the left side of a colon (:) within a JSON-like object.
+      Therefore, this expression must preceed the one for all LiteralStringDouble -->
+      <rule pattern="&#34;(\\.|[^\\&#34;\r\n])*&#34;(?=\s*:)">
+        <token type="NameTag"/>
+      </rule>
+      <rule pattern="&#34;(\\\\|\\&#34;|[^&#34;])*&#34;">
+        <token type="LiteralStringDouble"/>
+      </rule>
+      <rule pattern="&#39;(\\|\\&#39;|[^&#39;])*&#39;">
+        <token type="LiteralStringSingle"/>
+      </rule>
+      <rule pattern="`.*`">
+        <token type="LiteralStringBacktick"/>
+      </rule>
+      <!-- NOTE: This expression matches everything remaining, which should be only JSONata names.
+      Therefore, it has been left as last intentionally -->
+      <rule pattern="[a-zA-Z0-9_]*"> 
+        <token type="Name"/>
+      </rule>
+    </state>
+  </rules>
+</lexer>
\ No newline at end of file
diff --git a/lexers/testdata/jsonata.actual b/lexers/testdata/jsonata.actual
new file mode 100644
index 0000000..4a5eeff
--- /dev/null
+++ b/lexers/testdata/jsonata.actual
@@ -0,0 +1,38 @@
+(
+  $.a.b.c;
+
+  /* Example json input */
+  $jsonInput := {
+    "firstName": "JSONata",
+    "lastName": "User",
+    "age": 25,
+    "skills": ["programming", "cooking"],
+    "over 18": true,
+    "coordinates": { "latitude": 51.509865, "longitude": -0.118092 }
+  };
+  
+  /* Example function definition */
+  $getGreeting := function($firstName, $lastName) {
+    "Hello, " & $firstName & " " & $lastName
+  };
+
+  $isAdult := $jsonInput.`over 18`;
+  
+  /* Example json transformation */
+  $jsonInput.{
+    "fullName": $join([firstName, lastName], " "),
+    "greeting": $getGreeting(firstName, lastName),
+    "skills": $append(skills, "jsonata"),
+    "isAdult": $isAdult
+  }
+
+  $spreadOp = [1..5];
+  $sortOperator := Account.Order^(>Price, <Quantity);
+  $descendants := **.Postcode;
+  $math:= 10 / 5 * 2 + 4 - 2 % 2;
+  $conditions := 20 > 10 and 10 > 20 ? 20 = 20 or 20 != 10 : 10 in [1,10,20];
+  $chainOperator := " trim " ~> $trim();
+  $sum := +10 + -5 - +3 + $x / $y;
+  $multiplyVars := $a*$b * $c;
+  $wildcard := $.wild.*.card;
+)
\ No newline at end of file
diff --git a/lexers/testdata/jsonata.expected b/lexers/testdata/jsonata.expected
new file mode 100644
index 0000000..345e0ba
--- /dev/null
+++ b/lexers/testdata/jsonata.expected
@@ -0,0 +1,335 @@
+[
+  {"type":"Punctuation","value":"("},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"a"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"b"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"c"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"CommentMultiline","value":"/* Example json input */"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$jsonInput"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"firstName\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"JSONata\""},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"lastName\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"User\""},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"age\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"25"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"skills\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringDouble","value":"\"programming\""},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"cooking\""},
+  {"type":"Punctuation","value":"],"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"over 18\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordConstant","value":"true"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"coordinates\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":" "},
+  {"type":"NameTag","value":"\"latitude\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"51.509865"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"NameTag","value":"\"longitude\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"-0.118092"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Punctuation","value":"};"},
+  {"type":"Text","value":"\n  \n  "},
+  {"type":"CommentMultiline","value":"/* Example function definition */"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$getGreeting"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"function"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"$firstName"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$lastName"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"LiteralStringDouble","value":"\"Hello, \""},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u0026"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$firstName"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u0026"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\" \""},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u0026"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$lastName"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Punctuation","value":"};"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"NameVariable","value":"$isAdult"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$jsonInput"},
+  {"type":"Punctuation","value":"."},
+  {"type":"LiteralStringBacktick","value":"`over 18`"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n  \n  "},
+  {"type":"CommentMultiline","value":"/* Example json transformation */"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$jsonInput"},
+  {"type":"Punctuation","value":".{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"fullName\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"$join"},
+  {"type":"Punctuation","value":"(["},
+  {"type":"Name","value":"firstName"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"lastName"},
+  {"type":"Punctuation","value":"],"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\" \""},
+  {"type":"Punctuation","value":"),"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"greeting\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"$getGreeting"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"firstName"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"lastName"},
+  {"type":"Punctuation","value":"),"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"skills\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"$append"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"skills"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"jsonata\""},
+  {"type":"Punctuation","value":"),"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameTag","value":"\"isAdult\""},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$isAdult"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"NameVariable","value":"$spreadOp"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Punctuation","value":".."},
+  {"type":"LiteralNumberInteger","value":"5"},
+  {"type":"Punctuation","value":"];"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$sortOperator"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Account"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"Order"},
+  {"type":"Operator","value":"^"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Operator","value":"\u003e"},
+  {"type":"Name","value":"Price"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u003c"},
+  {"type":"Name","value":"Quantity"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$descendants"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"**"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"Postcode"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$math"},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"/"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"5"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"%"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$conditions"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"20"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"and"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"20"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"?"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"20"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"20"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"or"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"20"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"!="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumberInteger","value":"20"},
+  {"type":"Punctuation","value":"];"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$chainOperator"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\" trim \""},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"~\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"$trim"},
+  {"type":"Punctuation","value":"();"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$sum"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"+10"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"-5"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"+3"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$x"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"/"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$y"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$multiplyVars"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$a"},
+  {"type":"Operator","value":"*"},
+  {"type":"NameVariable","value":"$b"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$c"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameVariable","value":"$wildcard"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"wild"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Operator","value":"*"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"card"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":")"}
+]