Repository: lark-parser/lark
Pull request #829 (2021-02-06 01:24:23)

Fix for #818 + tests
Allows internal Transformers to return non-Tokens from TERMINAL callbacks, like when used as a separate step.

This is done by moving the handling from after lexing to after matching in `ParserState.feed_token`.

This PR still has a few problems, as shown by the failing tests:

- deserializing is broken since we now have a cycle dependency between creating the callbacks and deserializing the list of terminals. Do you know an easy fix?
- The new tests expose a bug introduced in #379, which makes all `Transformer_InPlace` instances passed as internal transformers behave like they have `@v_args(tree=True)`. I do not have an overview what the best fix here is.

For both I can take a look but maybe you have a simple idea how to fix it.


Patch:
diff --git a/lark/lark.py b/lark/lark.py
index b1b9270..c6f3f7e 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -185,7 +185,7 @@ class LarkOptions(Serialize):
 
 # Options that can be passed to the Lark parser, even when it was loaded from cache/standalone.
 # These option are only used outside of `load_grammar`.
-_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'}
+_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class'}
 
 _VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None)
 _VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest')
@@ -338,9 +338,7 @@ class Lark(Serialize):
                     rule.options.priority = None
 
         # TODO Deprecate lexer_callbacks?
-        lexer_callbacks = (_get_lexer_callbacks(self.options.transformer, self.terminals)
-                           if self.options.transformer
-                           else {})
+        lexer_callbacks = {}
         lexer_callbacks.update(self.options.lexer_callbacks)
 
         self.lexer_conf = LexerConf(self.terminals, re_module, self.ignore_tokens, self.options.postlex, lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes)
@@ -369,8 +367,7 @@ class Lark(Serialize):
         return TraditionalLexer(lexer_conf)
 
     def _prepare_callbacks(self):
-        self.parser_class = get_frontend(self.options.parser, self.options.lexer)
-        self._callbacks = None
+        self._callbacks = {}
         # we don't need these callbacks if we aren't building a tree
         if self.options.ambiguity != 'forest':
             self._parse_tree_builder = ParseTreeBuilder(
@@ -380,9 +377,11 @@ class Lark(Serialize):
                     self.options.parser != 'lalr' and self.options.ambiguity == 'explicit',
                     self.options.maybe_placeholders
                 )
-            self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
+            self._callbacks.update(self._parse_tree_builder.create_callback(self.options.transformer))
+        self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals))
 
     def _build_parser(self):
+        self.parser_class = get_frontend(self.options.parser, self.options.lexer)
         self._prepare_callbacks()
         parser_conf = ParserConf(self.rules, self._callbacks, self.options.start)
         return self.parser_class(self.lexer_conf, parser_conf, options=self.options)
@@ -422,16 +421,21 @@ class Lark(Serialize):
         self.options = LarkOptions.deserialize(options, memo)
         self.rules = [Rule.deserialize(r, memo) for r in data['rules']]
         self.source_path = '<deserialized>'
+        self.parser_class = get_frontend(self.options.parser, self.options.lexer)
+        self.lexer_conf = self.parser_class.deserialize_lexer_conf( # We need the terminals list to for _prepare_callbacks
+            data['parser'], 
+            memo, 
+            self.options)
+        self.terminals = self.lexer_conf.terminals
+        self._terminals_dict = {t.name: t for t in self.terminals}
         self._prepare_callbacks()
         self.parser = self.parser_class.deserialize(
             data['parser'],
             memo,
+            self.lexer_conf,
             self._callbacks,
             self.options,  # Not all, but multiple attributes are used
         )
-        self.lexer_conf = self.parser.lexer_conf
-        self.terminals = self.parser.lexer_conf.terminals
-        self._terminals_dict = {t.name: t for t in self.terminals}
         return self
 
     @classmethod
diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py
index 0fab159..e9d89b0 100644
--- a/lark/parser_frontends.py
+++ b/lark/parser_frontends.py
@@ -38,23 +38,25 @@ class MakeParsingFrontend:
         parser_conf.parser_type = self.parser_type
         lexer_conf.lexer_type = self.lexer_type
         return ParsingFrontend(lexer_conf, parser_conf, options)
-
+    
     @classmethod
-    def deserialize(cls, data, memo, callbacks, options):
+    def deserialize_lexer_conf(cls, data, memo, options):
+        # We need lexer_conf earley to have the terminals that we need to produce the callback list for paser_conf
+        # So we split deserialize into two methods
         lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo)
-        parser_conf = ParserConf.deserialize(data['parser_conf'], memo)
-        parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug)
-        parser_conf.callbacks = callbacks
-
-        terminals = [item for item in memo.values() if isinstance(item, TerminalDef)]
-
-        lexer_conf.callbacks = _get_lexer_callbacks(options.transformer, terminals)
+        lexer_conf.callbacks = options.lexer_callbacks or {}
         lexer_conf.re_module = regex if options.regex else re
         lexer_conf.use_bytes = options.use_bytes
         lexer_conf.g_regex_flags = options.g_regex_flags
         lexer_conf.skip_validation = True
         lexer_conf.postlex = options.postlex
+        return lexer_conf
 
+    @classmethod
+    def deserialize(cls, data, memo, lexer_conf, callbacks, options):
+        parser_conf = ParserConf.deserialize(data['parser_conf'], memo)
+        parser = LALR_Parser.deserialize(data['parser'], memo, callbacks, options.debug)
+        parser_conf.callbacks = callbacks
         return ParsingFrontend(lexer_conf, parser_conf, options, parser=parser)
 
 
diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py
index 9f08b81..5ec63aa 100644
--- a/lark/parsers/lalr_parser.py
+++ b/lark/parsers/lalr_parser.py
@@ -101,7 +101,7 @@ class ParserState(object):
                 # shift once and return
                 assert not is_end
                 state_stack.append(arg)
-                value_stack.append(token)
+                value_stack.append(token if token.type not in callbacks else callbacks[token.type](token))
                 return
             else:
                 # reduce+shift as many times as necessary


Test patch:
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 4a07d7a..d5a92c8 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -3,7 +3,7 @@ from __future__ import absolute_import
 import sys
 from unittest import TestCase, main
 
-from lark import Lark, Tree
+from lark import Lark, Tree, Transformer
 from lark.lexer import Lexer, Token
 import lark.lark as lark_module
 
@@ -48,6 +48,18 @@ class CustomLexer(Lexer):
             yield Token('A', obj)
 
 
+class TestT(Transformer):
+    def add(self, children):
+        return sum(children if isinstance(children, list) else children.children)
+
+    def NUM(self, token):
+        return int(token)
+
+
+def append_zero(t):
+    return t.update(value=t.value + '0')
+
+
 class TestCache(TestCase):
     def setUp(self):
         pass
@@ -73,7 +85,7 @@ class TestCache(TestCase):
             parser = Lark(g, parser='lalr', cache=True)
             assert parser.parse('a') == Tree('start', [])
 
-            parser = Lark(g+' "b"', parser='lalr', cache=True)
+            parser = Lark(g + ' "b"', parser='lalr', cache=True)
             assert len(mock_fs.files) == 2
             assert parser.parse('ab') == Tree('start', [])
 
@@ -92,10 +104,31 @@ class TestCache(TestCase):
             Lark(g, parser="lalr", debug=True, cache=True)
             parser = Lark(g, parser="lalr", debug=True, cache=True)
             assert parser.options.options['debug']
+
+            # Test inline transformer (tree-less) & lexer_callbacks
+            mock_fs.files = {}
+            g = """
+            start: add+
+            add: NUM "+" NUM
+            NUM: /\d+/
+            %ignore " "
+            """
+            text = "1+2 3+4"
+            expected = Tree('start', [30, 70])
+
+            parser = Lark(g, parser='lalr', transformer=TestT(), cache=True, lexer_callbacks={'NUM': append_zero})
+            res0 = parser.parse(text)
+            parser = Lark(g, parser='lalr', transformer=TestT(), cache=True, lexer_callbacks={'NUM': append_zero})
+            assert len(mock_fs.files) == 1
+            res1 = parser.parse(text)
+            res2 = TestT().transform(Lark(g, parser="lalr", cache=True, lexer_callbacks={'NUM': append_zero}).parse(text))
+            assert res0 == res1 == res2 == expected
+
+
+
         finally:
             lark_module.FS = fs
 
 
-
 if __name__ == '__main__':
     main()
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 8500b01..ef91a92 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -10,7 +10,7 @@ from copy import copy, deepcopy
 
 from lark.utils import Py36, isascii
 
-from lark import Token
+from lark import Token, Transformer_NonRecursive
 
 try:
     from cStringIO import StringIO as cStringIO
@@ -34,7 +34,7 @@ from lark import logger
 from lark.lark import Lark
 from lark.exceptions import GrammarError, ParseError, UnexpectedToken, UnexpectedInput, UnexpectedCharacters
 from lark.tree import Tree
-from lark.visitors import Transformer, Transformer_InPlace, v_args
+from lark.visitors import Transformer, Transformer_InPlace, v_args, Transformer_InPlaceRecursive
 from lark.grammar import Rule
 from lark.lexer import TerminalDef, Lexer, TraditionalLexer
 from lark.indenter import Indenter
@@ -162,6 +162,28 @@ class TestParsers(unittest.TestCase):
         r = p.parse("x")
         self.assertEqual( r.children, ["X!"] )
 
+    def test_visit_tokens2(self):
+        g = """
+        start: add+
+        add: NUM "+" NUM
+        NUM: /\d+/
+        %ignore " "
+        """
+        text = "1+2 3+4"
+        expected = Tree('start', [3, 7])
+        for base in (Transformer, Transformer_InPlace, Transformer_NonRecursive, Transformer_InPlaceRecursive):
+            class T(base):
+                def add(self, children):
+                    return sum(children if isinstance(children, list) else children.children)
+                
+                def NUM(self, token):
+                    return int(token)
+                
+            
+            parser = Lark(g, parser='lalr', transformer=T())
+            result = parser.parse(text)
+            self.assertEqual(result, expected)
+
     def test_vargs_meta(self):
 
         @v_args(meta=True)