Repository: jsonata-js/jsonata (js)

Issue:
Property names starting with digits
Using the JSONata exerciser at http://try.jsonata.org - not sure which version that is.

data {"7a":33, "a7":55, "7":88}:

7a -> Syntax error: “a”
“7a” -> “7a”
`7a` -> 33

$.7a -> Syntax error: “a”
$.“7a” -> 33
$.`7a` -> 33

but:

$.7 -> 88

It seems inconsistent to allow $.7 but not $.7a unquoted (or unticked).  Identifiers which start with a digit should be treated the same way regardless of whether they consist entirely of digits or only partly, in my opinion.  I prefer that they should need to be enclosed.



Pull request description:
disallow numeric and true/false/null literals from path steps
When literal values are used as 'steps' within a location path, they are converted to `name` tokens.
E.g. `msg."some property"` modifies the `literal` token to be a `name` token so that property names can contain reserved characters.  However, numeric literals and 'value' literals (i.e. `true`, `false` & `null`) are also affected by the same logic.  This leads to situations whereby an an expression such as `msg.7` attempts to select a property named '7'.  This was never intended and results from the fact that a `literal` AST node is derived from all of the `string`, `number` and `value` tokens.  This PR changes this behaviour such the that original token type is maintained through to the AST and an error is thrown if a literal number or value appears as a step in a path.  It also has a benefit in the evaluator whereby literal types to not have to be tested at runtime.

Resolves #147 

Patch:
diff --git a/jsonata.js b/jsonata.js
index 69d166c..4931613 100644
--- a/jsonata.js
+++ b/jsonata.js
@@ -719,7 +719,6 @@ var jsonata = (function() {
                 case 'string':
                 case 'number':
                 case 'value':
-                    type = "literal";
                     symbol = symbol_table["(literal)"];
                     break;
                 case 'regex':
@@ -1154,9 +1153,18 @@ var jsonata = (function() {
                                 rest = {type: 'path', steps: [rest]};
                             }
                             Array.prototype.push.apply(result.steps, rest.steps);
-                            // any steps within a path that are literals, should be changed to 'name'
+                            // any steps within a path that are string literals, should be changed to 'name'
                             result.steps.filter(function(step) {
-                                return step.type === 'literal';
+                                if(step.type === 'number' || step.type === 'value') {
+                                    // don't allow steps to be numbers or the values true/false/null
+                                    throw {
+                                        code: "S0213",
+                                        stack: (new Error()).stack,
+                                        position: step.position,
+                                        value: step.value
+                                    };
+                                }
+                                return step.type === 'string';
                             }).forEach(function(lit) {
                                 lit.type = 'name';
                             });
@@ -1261,7 +1269,7 @@ var jsonata = (function() {
                         // all other unary expressions - just process the expression
                         result.expression = ast_optimize(expr.expression);
                         // if unary minus on a number, then pre-process
-                        if (expr.value === '-' && result.expression.type === 'literal' && isNumeric(result.expression.value)) {
+                        if (expr.value === '-' && result.expression.type === 'number') {
                             result = result.expression;
                             result.value = -result.value;
                         }
@@ -1311,7 +1319,9 @@ var jsonata = (function() {
                         result.keepSingletonArray = true;
                     }
                     break;
-                case 'literal':
+                case 'string':
+                case 'number':
+                case 'value':
                 case 'wildcard':
                 case 'descendant':
                 case 'variable':
@@ -1472,7 +1482,9 @@ var jsonata = (function() {
             case 'name':
                 result = evaluateName(expr, input, environment);
                 break;
-            case 'literal':
+            case 'string':
+            case 'number':
+            case 'value':
                 result = evaluateLiteral(expr, input, environment);
                 break;
             case 'wildcard':
@@ -1706,7 +1718,7 @@ var jsonata = (function() {
                 inputSequence = createSequence(inputSequence);
             }
             results = createSequence();
-            if (predicate.type === 'literal' && isNumeric(predicate.value)) {
+            if (predicate.type === 'number') {
                 var index = predicate.value;
                 if (!Number.isInteger(index)) {
                     // round it down
@@ -4805,6 +4817,7 @@ var jsonata = (function() {
         "S0210": "Each step can only have one grouping expression",
         "S0211": "The symbol {{token}} cannot be used as a unary operator",
         "S0212": "The left side of := must be a variable name (start with $)",
+        "S0213": "The literal value {{value}} cannot be used as a step within a path expression",
         "S0301": "Empty regular expressions are not allowed",
         "S0302": "No terminating / in regular expression",
         "S0402": "Choice groups containing parameterized types are not supported",


Test patch:
diff --git a/test/parser-recovery.js b/test/parser-recovery.js
index 53a7be0..d8995e1 100644
--- a/test/parser-recovery.js
+++ b/test/parser-recovery.js
@@ -25,7 +25,7 @@ describe('Invoke parser with valid expression', function() {
                         "predicate": [
                             {
                                 "value": 0,
-                                "type": "literal",
+                                "type": "number",
                                 "position": 15
                             }
                         ]
@@ -142,7 +142,7 @@ describe('Invoke parser with incomplete expression', function() {
                                 "token": ";",
                                 "position": 15,
                                 "remaining": [
-                                    {"value": 0, "type": "literal", "position": 16},
+                                    {"value": 0, "type": "number", "position": 16},
                                     {"type": "operator", "value": "]", "position": 17},
                                     {"type": "operator", "value": ".", "position": 18},
                                     {"type": "name", "value": "Product", "position": 25}
@@ -160,7 +160,7 @@ describe('Invoke parser with incomplete expression', function() {
                     "token": ";",
                     "position": 15,
                     "remaining": [
-                        {"value": 0, "type": "literal", "position": 16},
+                        {"value": 0, "type": "number", "position": 16},
                         {"type": "operator", "value": "]", "position": 17},
                         {"type": "operator", "value": ".", "position": 18},
                         {"type": "name", "value": "Product", "position": 25}
@@ -175,7 +175,7 @@ describe('Invoke parser with incomplete expression', function() {
                     "remaining": [
                         {
                             "value": 0,
-                            "type": "literal",
+                            "type": "number",
                             "position": 16
                         }
                     ]
@@ -205,7 +205,7 @@ describe('Invoke parser with incomplete expression', function() {
                         "predicate": [
                             {
                                 "value": 0,
-                                "type": "literal",
+                                "type": "number",
                                 "position": 15
                             }
                         ]
@@ -251,7 +251,7 @@ describe('Invoke parser with incomplete expression', function() {
                         "predicate": [
                             {
                                 "value": 0,
-                                "type": "literal",
+                                "type": "number",
                                 "position": 15
                             }
                         ]
@@ -301,7 +301,7 @@ describe('Invoke parser with incomplete expression', function() {
                             "predicate": [
                                 {
                                     "value": 0,
-                                    "type": "literal",
+                                    "type": "number",
                                     "position": 14
                                 }
                             ]
diff --git a/test/test-suite/groups/token-conversion/case000.json b/test/test-suite/groups/token-conversion/case000.json
new file mode 100644
index 0000000..2c52d74
--- /dev/null
+++ b/test/test-suite/groups/token-conversion/case000.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$.'7a'",
+    "data": {"7a":33, "a7":55, "7":88},
+    "bindings": {},
+    "result": 33
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/token-conversion/case001.json b/test/test-suite/groups/token-conversion/case001.json
new file mode 100644
index 0000000..ec3ba99
--- /dev/null
+++ b/test/test-suite/groups/token-conversion/case001.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$.'7'",
+    "data": {"7a":33, "a7":55, "7":88},
+    "bindings": {},
+    "result": 88
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/token-conversion/case002.json b/test/test-suite/groups/token-conversion/case002.json
new file mode 100644
index 0000000..f37359b
--- /dev/null
+++ b/test/test-suite/groups/token-conversion/case002.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$.7a",
+    "data": {"7a":33, "a7":55, "7":88},
+    "bindings": {},
+    "code": "S0201"
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/token-conversion/case003.json b/test/test-suite/groups/token-conversion/case003.json
new file mode 100644
index 0000000..1679f28
--- /dev/null
+++ b/test/test-suite/groups/token-conversion/case003.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$.7",
+    "data": {"7a":33, "a7":55, "7":88},
+    "bindings": {},
+    "code": "S0213"
+}
\ No newline at end of file