Repository: lark-parser/lark
Pull request #930 (2021-06-29 20:35:19)

fix tree_matcher when keep_all_tokens=True by setting sym.filter_out …
…correctly.

Previously `filter_out` might have the wrong value. Inside of tree_matcher (and especially reconstructor) the rule options are not accessible. The easiest fix is to just make sure `sym.filter_out` is correct.

Failing example:

```
start: _A
_A: "a"
```

Wouldn't get reconstructed when `keep_all_tokens=True`


Patch:
diff --git a/lark-stubs/reconstruct.pyi b/lark-stubs/reconstruct.pyi
index a8d39e3..9826428 100644
--- a/lark-stubs/reconstruct.pyi
+++ b/lark-stubs/reconstruct.pyi
@@ -34,6 +34,5 @@ class Reconstructor:
     def __init__(self, parser: Lark, term_subs: Dict[str, Callable[[Symbol], str]] = ...):
         ...
 
-    def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]=None,
-                    insert_spaces: bool = True) -> str:
+    def reconstruct(self, tree: Tree, postproc: Callable[[Iterable[str]], Iterable[str]]) -> str:
         ...
diff --git a/lark-stubs/tree.pyi b/lark-stubs/tree.pyi
index ea99ff6..98aadff 100644
--- a/lark-stubs/tree.pyi
+++ b/lark-stubs/tree.pyi
@@ -40,7 +40,7 @@ class Tree:
     def expand_kids_by_index(self, *indices: int) -> None:
         ...
 
-    def scan_values(self, pred: Callable[[Union[str, Tree]], bool]) -> Iterator[str]:
+    def scan_values(self, pred: Callable[[Union[str, Tree]], bool]) -> List[str]:
         ...
 
     def iter_subtrees(self) -> Iterator[Tree]:
diff --git a/lark/lexer.py b/lark/lexer.py
index 4062c2d..a2aefd2 100644
--- a/lark/lexer.py
+++ b/lark/lexer.py
@@ -120,7 +120,7 @@ class Token(Str):
     Attributes:
         type: Name of the token (as specified in grammar)
         value: Value of the token (redundant, as ``token.value == token`` will always be true)
-        start_pos: The index of the token in the text
+        pos_in_stream: The index of the token in the text
         line: The line of the token in the text (starting with 1)
         column: The column of the token in the text (starting with 1)
         end_line: The line where the token ends
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index dbf4a1f..c7b98a7 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -330,7 +330,7 @@ class PrepareAnonTerminals(Transformer_InPlace):
                 try:
                     term_name = _TERMINAL_NAMES[value]
                 except KeyError:
-                    if value and is_id_continue(value) and is_id_start(value[0]) and value.upper() not in self.term_set:
+                    if is_id_continue(value) and is_id_start(value[0]) and value.upper() not in self.term_set:
                         term_name = value.upper()
 
                 if term_name in self.term_set:
@@ -448,9 +448,6 @@ def _literal_to_pattern(literal):
 
     s = eval_escaping(x)
 
-    if s == "":
-        raise GrammarError("Empty terminals are not allowed (%s)" % literal)
-
     if literal.type == 'STRING':
         s = s.replace('\\\\', '\\')
         return PatternStr(s, flags, raw=literal.value)
@@ -633,7 +630,10 @@ class Grammar:
                 else:
                     exp_options = options
 
-                assert all(isinstance(x, Symbol) for x in expansion), expansion
+                for sym in expansion:
+                    assert isinstance(sym, Symbol)
+                    if sym.is_term and exp_options and exp_options.keep_all_tokens:
+                        sym.filter_out = False
                 rule = Rule(NonTerminal(name), expansion, i, alias, exp_options)
                 compiled_rules.append(rule)
 
diff --git a/lark/reconstruct.py b/lark/reconstruct.py
index ab2fb38..2efc0ae 100644
--- a/lark/reconstruct.py
+++ b/lark/reconstruct.py
@@ -87,14 +87,14 @@ class Reconstructor(TreeMatcher):
             else:
                 yield item
 
-    def reconstruct(self, tree, postproc=None, insert_spaces=True):
+    def reconstruct(self, tree, postproc=None):
         x = self._reconstruct(tree)
         if postproc:
             x = postproc(x)
         y = []
         prev_item = ''
         for item in x:
-            if insert_spaces and prev_item and item and is_id_continue(prev_item[-1]) and is_id_continue(item[0]):
+            if prev_item and item and is_id_continue(prev_item[-1]) and is_id_continue(item[0]):
                 y.append(' ')
             y.append(item)
             prev_item = item
diff --git a/lark/utils.py b/lark/utils.py
index b9d7ac3..70516e6 100644
--- a/lark/utils.py
+++ b/lark/utils.py
@@ -241,6 +241,17 @@ except ImportError:
             pass
 
 
+try:
+    compare = cmp
+except NameError:
+    def compare(a, b):
+        if a == b:
+            return 0
+        elif a > b:
+            return 1
+        return -1
+
+
 class Enumerator(Serialize):
     def __init__(self):
         self.enums = {}


Test patch:
diff --git a/tests/test_grammar.py b/tests/test_grammar.py
index a643117..6a1aefa 100644
--- a/tests/test_grammar.py
+++ b/tests/test_grammar.py
@@ -22,10 +22,6 @@ class TestGrammar(TestCase):
                 else:
                     assert False, "example did not raise an error"
 
-    def test_empty_literal(self):
-        # Issues #888
-        self.assertRaises(GrammarError, Lark, "start: \"\"")
-
     def test_override_rule(self):
         # Overrides the 'sep' template in existing grammar to add an optional terminating delimiter
         # Thus extending it beyond its original capacity
diff --git a/tests/test_reconstructor.py b/tests/test_reconstructor.py
index f132312..e2f2dbe 100644
--- a/tests/test_reconstructor.py
+++ b/tests/test_reconstructor.py
@@ -3,6 +3,7 @@
 import json
 import sys
 import unittest
+from itertools import product
 from unittest import TestCase
 
 from lark import Lark
@@ -20,8 +21,8 @@ def _remove_ws(s):
 
 class TestReconstructor(TestCase):
 
-    def assert_reconstruct(self, grammar, code):
-        parser = Lark(grammar, parser='lalr', maybe_placeholders=False)
+    def assert_reconstruct(self, grammar, code, **options):
+        parser = Lark(grammar, parser='lalr', maybe_placeholders=False, **options)
         tree = parser.parse(code)
         new = Reconstructor(parser).reconstruct(tree)
         self.assertEqual(_remove_ws(code), _remove_ws(new))
@@ -142,6 +143,17 @@ class TestReconstructor(TestCase):
         new_json = Reconstructor(json_parser).reconstruct(tree)
         self.assertEqual(json.loads(new_json), json.loads(test_json))
 
+    def test_keep_all_tokens(self):
+        g = """
+        start: "a"? _B? c? _d?
+        _B: "b"
+        c: "c"
+        _d: "d"
+        """
+        examples = list(map(''.join, product(('', 'a'), ('', 'b'), ('', 'c'), ('', 'd'), )))
+        for code in examples:
+            self.assert_reconstruct(g, code, keep_all_tokens=True)
+
     @unittest.skipIf(sys.version_info < (3, 0), "Python 2 does not play well with Unicode.")
     def test_switch_grammar_unicode_terminal(self):
         """