Repository: neogeny/TatSu

Issue:
Literal ``None`` is interpreted as a string
When using a construct like
```
null = 'null' value:`None` ;
```
`value` takes the string value `"None"` and not `None`.  `` `None` ``is used in the Tatsu's EBNF grammar in a few places and I have the impression that it is meant to be the `None` object no the `"None"` string. I find it mildly confusing that bot `` `"foo"` `` and `` `foo` `` are interpreted as strings, but I don't think that this can be changed, for backward compatibility. 

Discussion:

Patch:
diff --git a/Makefile b/Makefile
index c46b857..839260b 100644
--- a/Makefile
+++ b/Makefile
@@ -35,7 +35,7 @@ pylint:
 
 
 mypy:
-	mypy   --ignore-missing-imports .
+	mypy   --ignore-missing-imports . --exclude dist
 
 
 clean:
diff --git a/docs/syntax.rst b/docs/syntax.rst
index 11095fe..6c8da02 100644
--- a/docs/syntax.rst
+++ b/docs/syntax.rst
@@ -367,13 +367,15 @@ semantic action. For example::
 
   boolean_option = name ['=' (boolean|`true`) ] ;
 
-String interpolation in the style of ``str.format()`` over the names
-in the current `AST`_ is applied for *constant* elements. Occurrences of
-the ``{`` character must be scaped to ``\{`` if they are not intended
-for interpolation. A *constant* expression that hast type ``str`` is
-evaluated using::
 
-  eval(f'{repr(constant)}.format(**{ast})')
+If the text evaluates to a Python literal (with ``ast.literal_eval()``), that
+will be the returned value.  Otherwise, string interpolation in the style of
+``str.format()`` over the names in the current `AST`_ is applied for
+*constant* elements. Occurrences of the ``{`` character must be scaped to
+``\{`` if they are not intended for interpolation. A *constant* expression
+that hast type ``str`` is evaluated using::
+
+  eval(f'{"f" + repr(text)}', {}, ast)
 
 
 `````constant`````
diff --git a/grammar/tatsu.ebnf b/grammar/tatsu.ebnf
index 91b5774..3e18f25 100644
--- a/grammar/tatsu.ebnf
+++ b/grammar/tatsu.ebnf
@@ -391,12 +391,8 @@ name
 
 constant::Constant
     =
-    &"`"
-    (
-        | "```" ~ @:/(?ms)((?:.|\n)*?)(?:```)/
-        | "`" @:literal "`"
-        | /`(.*?)`/
-    )
+    | /(?ms)```(.|\n)*?```/
+    | /`(.*?)`/
     ;
 
 
@@ -414,7 +410,7 @@ token::Token
 
 literal
     =
-    string | raw_string | boolean | word | hex | float | int
+    string | raw_string | boolean | word | hex | float | int | null
     ;
 
 
@@ -500,6 +496,12 @@ boolean
     ;
 
 
+ null
+    =
+    'None'
+    ;
+
+
 eof::EOF
     =
     '$' ~
diff --git a/tatsu/contexts.py b/tatsu/contexts.py
index 7c186fc..7806213 100644
--- a/tatsu/contexts.py
+++ b/tatsu/contexts.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sys
+import ast as stdlib_ast
 from typing import Type
 import functools
 from contextlib import contextmanager
@@ -19,7 +20,7 @@ from . import color
 from .ast import AST
 from .collections import OrderedSet as oset
 from .util import prune_dict, is_list, info, safe_name
-from .util import left_assoc, right_assoc
+from .util import left_assoc, right_assoc, trim
 from .tokenizing import Tokenizer
 from .infos import (
     Alert,
@@ -380,8 +381,6 @@ class ParseContext:
 
     def _append_cst(self, node):
         self.last_node = node
-        if node is None:
-            return
         previous = self.cst
         if previous is None:
             self.cst = self._copy_node(node)
@@ -733,6 +732,13 @@ class ParseContext:
     def _constant(self, literal):
         self._next_token()
         self._trace_match(literal)
+        if isinstance(literal, str):
+            try:
+                literal = stdlib_ast.literal_eval(literal.strip())
+            except (ValueError, SyntaxError):
+                if '\n' in literal:
+                    literal = trim(literal)
+                literal = eval(f'{"f" + repr(literal)}', {}, self.ast)  # pylint: disable=eval-used
         self._append_cst(literal)
         return literal
 
diff --git a/tatsu/grammars.py b/tatsu/grammars.py
index 64c5e28..127e41f 100644
--- a/tatsu/grammars.py
+++ b/tatsu/grammars.py
@@ -301,13 +301,7 @@ class Constant(Model):
         self.literal = self.ast
 
     def parse(self, ctx):
-        if isinstance(self.literal, str):
-            text = self.literal
-            if '\n' in text:
-                text = trim(text)
-            return eval(f'{"f" + repr(text)}', {}, dict(ctx.ast))  # pylint: disable=eval-used
-        else:
-            return self.literal
+        return ctx._constant(self.literal)
 
     def _first(self, k, f):
         return {()}


Test patch:
diff --git a/test/grammar/alerts_test.py b/test/grammar/alerts_test.py
index 263711d..e583e58 100644
--- a/test/grammar/alerts_test.py
+++ b/test/grammar/alerts_test.py
@@ -9,4 +9,5 @@ def test_alert_interpolation():
             start = a:number b: number i:^`"seen: {a}, {b}"` $ ;
             number = /\d+/ ;
     '''
-    assert parse(grammar, input) == {'a': '42', 'b': '69', 'i': 'seen: 42, 69'}
+    ast = parse(grammar, input)
+    assert ast == {'a': '42', 'b': '69', 'i': 'seen: 42, 69'}
diff --git a/test/grammar/constants_test.py b/test/grammar/constants_test.py
index 5e41686..32d36f4 100644
--- a/test/grammar/constants_test.py
+++ b/test/grammar/constants_test.py
@@ -36,3 +36,18 @@ def test_constant_interpolation_multiline():
     result = parse(grammar, input)
     print(result)
     assert result == {'a': '42', 'b': '69', 'i': 'seen:\n42\n69\n'}
+
+
+def test_evaluate_constant():
+    grammar = '''
+        @@grammar :: constants
+        start = int bool str null 'a' $;
+
+        int = `42` ;
+        bool = `True` ;
+        str = `Something` ;
+        null = `None` ;
+    '''
+
+    ast = parse(grammar, 'a')
+    assert ast == (42, True, 'Something', None, 'a')