Repository: lark-parser/lark

Issue:
"Unused" terminals are filtered out before post-lexer can handle them
Over a year ago, I implemented parsing for a custom language with significant whitespace.  I used a custom post-lexer (written by me) to transform certain tokens (e.g. newline and following whitespace) into new tokens (e.g. indent, unindent) that are used in the grammar.  This worked fine with the version of lark I was using at that time.  When I resumed development with the current version of lark, my code no longer worked.  I discovered that the source of my problems is commit:

145d8a8e7c64046bc7938d7a5e4dfcecfbbcb6eb

which is associated with issue #309 

The problem is that I have some terminals that are not used directly in the grammar, but are used by the post-lexer.  Filtering these terminals out breaks everything.  I was wondering how I should handle this?  Is there some way to tell lark to not filter out certain terminals that appear to be unused.

Thank you for any advice you can provide.

-Jeff


Discussion:

Patch:
diff --git a/lark-stubs/__init__.pyi b/lark-stubs/__init__.pyi
index 7545f2d..c010a93 100644
--- a/lark-stubs/__init__.pyi
+++ b/lark-stubs/__init__.pyi
@@ -5,5 +5,7 @@ from .visitors import *
 from .exceptions import *
 from .lexer import *
 from .lark import *
+from logging import Logger as _Logger
 
+logger: _Logger
 __version__: str = ...
diff --git a/lark-stubs/lark.pyi b/lark-stubs/lark.pyi
index c39ae3d..f538dbb 100644
--- a/lark-stubs/lark.pyi
+++ b/lark-stubs/lark.pyi
@@ -2,7 +2,7 @@
 
 from typing import (
     TypeVar, Type, List, Dict, IO, Iterator, Callable, Union, Optional,
-    Literal, Protocol,
+    Literal, Protocol, Iterable,
 )
 from .visitors import Transformer
 from .lexer import Token, Lexer, TerminalDef
@@ -14,6 +14,8 @@ class PostLex(Protocol):
 
     def process(self, stream: Iterator[Token]) -> Iterator[Token]:
         ...
+    
+    always_accept: Iterable[str]
 
 
 class LarkOptions:
diff --git a/lark/lark.py b/lark/lark.py
index e749138..d1e19ca 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -269,8 +269,13 @@ class Lark(Serialize):
         # Parse the grammar file and compose the grammars (TODO)
         self.grammar = load_grammar(grammar, self.source, re_module)
 
+        if self.options.postlex is not None:
+            terminals_to_keep = set(self.options.postlex.always_accept)
+        else:
+            terminals_to_keep = set()
+
         # Compile the EBNF grammar into BNF
-        self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start)
+        self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep)
 
         if self.options.edit_terminals:
             for t in self.terminals:
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index cd36e4b..1590665 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -526,7 +526,7 @@ class Grammar:
         self.rule_defs = rule_defs
         self.ignore = ignore
 
-    def compile(self, start):
+    def compile(self, start, terminals_to_keep):
         # We change the trees in-place (to support huge grammars)
         # So deepcopy allows calling compile more than once.
         term_defs = deepcopy(list(self.term_defs))
@@ -641,7 +641,7 @@ class Grammar:
         used_terms = {t.name for r in compiled_rules
                              for t in r.expansion
                              if isinstance(t, Terminal)}
-        terminals, unused = classify_bool(terminals, lambda t: t.name in used_terms or t.name in self.ignore)
+        terminals, unused = classify_bool(terminals, lambda t: t.name in used_terms or t.name in self.ignore or t.name in terminals_to_keep)
         if unused:
             logger.debug("Unused terminals: %s", [t.name for t in unused])
 
diff --git a/lark/tree_matcher.py b/lark/tree_matcher.py
index b9306c4..8c1f17a 100644
--- a/lark/tree_matcher.py
+++ b/lark/tree_matcher.py
@@ -81,7 +81,8 @@ class TreeMatcher:
     def __init__(self, parser):
         # XXX TODO calling compile twice returns different results!
         assert parser.options.maybe_placeholders == False
-        self.tokens, rules, _extra = parser.grammar.compile(parser.options.start)
+        # XXX TODO: we just ignore the potential existence of a postlexer
+        self.tokens, rules, _extra = parser.grammar.compile(parser.options.start, set())
 
         self.rules_for_root = defaultdict(list)
 


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 83336c5..cea40b9 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1781,6 +1781,29 @@ def _make_parser_test(LEXER, PARSER):
             %import bad_test.NUMBER
             """
             self.assertRaises(IOError, _Lark, grammar)
+        
+        @unittest.skipIf(LEXER=='dynamic', "%declare/postlex doesn't work with dynamic")
+        def test_postlex_declare(self): # Note: this test does a lot. maybe split it up?
+            class TestPostLexer:
+                def process(self, stream):
+                    for t in stream:
+                        if t.type == 'A':
+                            t.type = 'B'
+                            yield t
+                        else:
+                            yield t
+
+                always_accept = ('A',)
+
+            parser = _Lark("""
+            start: B
+            A: "A"
+            %declare B
+            """, postlex=TestPostLexer())
+
+            test_file = "A"
+            tree = parser.parse(test_file)
+            self.assertEqual(tree.children, [Token('B', 'A')])
 
         @unittest.skipIf(PARSER != 'earley', "Currently only Earley supports priority in rules")
         def test_earley_prioritization(self):