Repository: neogeny/TatSu

Issue:
Allow False as a value for @@whitespace
Currently using this inside my grammar:
```
@@whitespace :: /(?!)/
```
which doesn't match anything.
```
@@whitespace :: False
```
would be more descriptive and probably more performant. `True` would set it to the default value.
Alternatively it could accept a string and then convert it to a regex, just like it does when calling parse with `whitespace=...`

Discussion:

Patch:
diff --git a/grammar/tatsu.ebnf b/grammar/tatsu.ebnf
index 6ff697e..f017615 100644
--- a/grammar/tatsu.ebnf
+++ b/grammar/tatsu.ebnf
@@ -25,9 +25,13 @@ directive
     =
     '@@' !'keyword' ~
     (
-            name:('comments' | 'eol_comments' | 'whitespace') ~
+            name:('comments' | 'eol_comments') ~
             ~
             '::' ~ value:regex
+        |
+            name:('whitespace') ~
+            ~
+            '::' ~ value:(regex|'None'|'False'|`None`)
         |
             name:('nameguard' | 'ignorecase' | 'left_recursion' | 'parseinfo')
             ~
@@ -40,7 +44,7 @@ directive
             name:('namechars')
             ~
             '::' ~ value:string
-    )
+    ) ~
     ;
 
 
diff --git a/tatsu/bootstrap.py b/tatsu/bootstrap.py
index b8db14f..88103ff 100644
--- a/tatsu/bootstrap.py
+++ b/tatsu/bootstrap.py
@@ -18,6 +18,7 @@ import sys
 from tatsu.buffering import Buffer
 from tatsu.parsing import Parser
 from tatsu.parsing import tatsumasu, leftrec, nomemo
+from tatsu.parsing import leftrec, nomemo  # noqa
 from tatsu.util import re, generic_main  # noqa
 
 
@@ -120,8 +121,6 @@ class EBNFBootstrapParser(Parser):
                                 self._token('comments')
                             with self._option():
                                 self._token('eol_comments')
-                            with self._option():
-                                self._token('whitespace')
                             self._error('no available options')
                     self.name_last_node('name')
                     self._cut()
@@ -130,6 +129,26 @@ class EBNFBootstrapParser(Parser):
                     self._cut()
                     self._regex_()
                     self.name_last_node('value')
+                with self._option():
+                    with self._group():
+                        self._token('whitespace')
+                    self.name_last_node('name')
+                    self._cut()
+                    self._cut()
+                    self._token('::')
+                    self._cut()
+                    with self._group():
+                        with self._choice():
+                            with self._option():
+                                self._regex_()
+                            with self._option():
+                                self._token('None')
+                            with self._option():
+                                self._token('False')
+                            with self._option():
+                                self._constant('None')
+                            self._error('no available options')
+                    self.name_last_node('value')
                 with self._option():
                     with self._group():
                         with self._choice():
@@ -174,6 +193,7 @@ class EBNFBootstrapParser(Parser):
                     self._string_()
                     self.name_last_node('value')
                 self._error('no available options')
+        self._cut()
         self.ast._define(
             ['name', 'value'],
             []
diff --git a/tatsu/codegen/python.py b/tatsu/codegen/python.py
index 13bbf13..f02c55b 100644
--- a/tatsu/codegen/python.py
+++ b/tatsu/codegen/python.py
@@ -495,6 +495,7 @@ class Grammar(Base):
                 from tatsu.buffering import Buffer
                 from tatsu.parsing import Parser
                 from tatsu.parsing import tatsumasu, leftrec, nomemo
+                from tatsu.parsing import leftrec, nomemo  # noqa
                 from tatsu.util import re, generic_main  # noqa
 
 
diff --git a/tatsu/parser_semantics.py b/tatsu/parser_semantics.py
index 3d8daf8..f0a16ea 100644
--- a/tatsu/parser_semantics.py
+++ b/tatsu/parser_semantics.py
@@ -121,6 +121,10 @@ class EBNFGrammarSemantics(ModelBuilderSemantics):
     def grammar(self, ast, *args):
         directives = OrderedDict((d.name, d.value) for d in ast.directives)
         keywords = set(ast.keywords or [])
+
+        if directives.get('whitespace') in ('None', 'False'):
+            directives['whitespace'] = ''
+
         return grammars.Grammar(
             self.grammar_name,
             list(self.rules.values()),


Test patch:
diff --git a/test/grammar/directive_test.py b/test/grammar/directive_test.py
index b91e86e..530def0 100644
--- a/test/grammar/directive_test.py
+++ b/test/grammar/directive_test.py
@@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera
 import unittest
 
 import tatsu
+from tatsu.exceptions import FailedParse
 from tatsu.util import trim
 from tatsu.codegen import codegen
 
@@ -23,6 +24,31 @@ class DirectiveTests(unittest.TestCase):
         code = codegen(model)
         compile('test.py', code, EXEC)
 
+    def test_whitespace_none_directive(self):
+        grammars = [
+            '''
+                @@whitespace :: None
+                @@nameguard :: False
+
+                test = {'x'}+ $;
+            ''',
+            '''
+                @@whitespace :: False
+                @@nameguard :: False
+
+                test = {'x'}+ $;
+            ''',
+        ]
+        for grammar in grammars:
+            model = tatsu.compile(grammar, "test")
+            self.assertEquals(['x', 'x'], model.parse('xx', trace=True))
+            try:
+                self.assertEquals(['x', 'x'], model.parse('x x', trace=True))
+            except FailedParse:
+                pass
+            else:
+                self.fail('parsed through non-whitespace')
+
     def test_eol_comments_re_directive(self):
         grammar = '''
             @@eol_comments :: /#.*?$/