Repository: lark-parser/lark
Pull request #1036 (2021-11-16 08:59:33)

Fix for Transformer.__default__ not called in tree-less LALR mode (Issue #1029)
Example of usage:

```python
from lark import Lark, Transformer

BUILD_TREE = False  # Change to False to try tree-less LALR mode

grammar = r"""
    start: expr

    expr: A B
        | A expr B

    A: "a"
    B: "b"

    %import common.WS
    %ignore WS
"""

class AbTransformer(Transformer):
    def __default__(self, data, children, meta):
        print(f'__default__({data!r}, {children!r}, {meta!r})')
        return data

def parse(x):
    if BUILD_TREE:
        parser = Lark(grammar, parser='lalr')
        return AbTransformer().transform(parser.parse(x))
    else:
        parser = Lark(grammar, parser='lalr', transformer=AbTransformer())
        return parser.parse(x)

print(parse('a a a b b b'))  # should print "start"
```


Patch:
diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py
index b2f462c..888cc73 100644
--- a/lark/parse_tree_builder.py
+++ b/lark/parse_tree_builder.py
@@ -352,6 +352,13 @@ class ParseTreeBuilder:
     def create_callback(self, transformer=None):
         callbacks = {}
 
+        default_handler = getattr(transformer, '__default__', None)
+        if default_handler:
+            def default_callback(data, children):
+                return default_handler(data, children, None)
+        else:
+            default_callback = self.tree_class
+
         for rule, wrapper_chain in self.rule_builders:
 
             user_callback_name = rule.alias or rule.options.template_source or rule.origin.name
@@ -363,7 +370,7 @@ class ParseTreeBuilder:
                 elif isinstance(transformer, Transformer_InPlace):
                     f = inplace_transformer(f)
             except AttributeError:
-                f = partial(self.tree_class, user_callback_name)
+                f = partial(default_callback, user_callback_name)
 
             for w in wrapper_chain:
                 f = w(f)


Test patch:
diff --git a/tests/test_cache.py b/tests/test_cache.py
index dd67435..5778632 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -112,7 +112,7 @@ class TestCache(TestCase):
         # Test inline transformer (tree-less) & lexer_callbacks
         # Note: the Transformer should not be saved to the file,
         #       and is made unpickable to check for that
-        g = r"""
+        g = """
         start: add+
         add: NUM "+" NUM
         NUM: /\d+/
diff --git a/tests/test_nearley/test_nearley.py b/tests/test_nearley/test_nearley.py
index c1783fe..a205446 100644
--- a/tests/test_nearley/test_nearley.py
+++ b/tests/test_nearley/test_nearley.py
@@ -16,7 +16,7 @@ NEARLEY_PATH = os.path.join(TEST_PATH, 'nearley')
 BUILTIN_PATH = os.path.join(NEARLEY_PATH, 'builtin')
 
 if not os.path.exists(BUILTIN_PATH):
-    logger.warning("Nearley not included. Skipping Nearley tests! (use git submodule to add)")
+    logger.warn("Nearley not included. Skipping Nearley tests! (use git submodule to add)")
     raise ImportError("Skipping Nearley tests!")
 
 import js2py    # Ensures that js2py exists, to avoid failing tests
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 32f6d16..7a76570 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -186,7 +186,7 @@ class TestParsers(unittest.TestCase):
         g = """
         start: add+
         add: NUM "+" NUM
-        NUM: /\\d+/
+        NUM: /\d+/
         %ignore " "
         """
         text = "1+2 3+4"
@@ -2560,6 +2560,32 @@ def _make_parser_test(LEXER, PARSER):
             s = "[0 1, 2,@, 3,,, 4, 5 6 ]$"
             tree = g.parse(s, on_error=ignore_errors)
 
+        @unittest.skipIf(PARSER!='lalr', "Tree-less mode is only supported in lalr")
+        def test_default_in_treeless_mode(self):
+            grammar = r"""
+                start: expr
+
+                expr: A B
+                    | A expr B
+
+                A: "a"
+                B: "b"
+
+                %import common.WS
+                %ignore WS
+            """
+            s = 'a a a b b b'
+
+            class AbTransformer(Transformer):
+                def __default__(self, data, children, meta):
+                    return '@', data, children
+
+            parser = _Lark(grammar)
+            a = AbTransformer().transform(parser.parse(s))
+            parser = _Lark(grammar, transformer=AbTransformer())
+            b = parser.parse(s)
+            assert a == b
+
 
     _NAME = "Test" + PARSER.capitalize() + LEXER.capitalize()
     _TestParser.__name__ = _NAME
diff --git a/tests/test_reconstructor.py b/tests/test_reconstructor.py
index fe1545e..4df1cb9 100644
--- a/tests/test_reconstructor.py
+++ b/tests/test_reconstructor.py
@@ -78,7 +78,7 @@ class TestReconstructor(TestCase):
         stmt: var op var
         !op: ("+" | "-" | "*" | "/")
         var: WORD
-        NL: /(\\r?\\n)+\\s*/
+        NL: /(\\r?\\n)+\s*/
         """ + common
 
         code = """
@@ -93,7 +93,7 @@ class TestReconstructor(TestCase):
         ?mult_stmt: sum_stmt ["*" sum_stmt]
         ?sum_stmt: var ["+" var]
         var: WORD
-        NL: /(\\r?\\n)+\\s*/
+        NL: /(\\r?\\n)+\s*/
         """ + common
 
         code = ['a', 'a*b', 'a+b', 'a*b+c', 'a+b*c', 'a+b*c+d']
@@ -168,7 +168,7 @@ class TestReconstructor(TestCase):
         stmt: "keyword" var op var
         !op: ("+=" | "-=" | "*=" | "/=")
         var: WORD
-        NL: /(\\r?\\n)+\\s*/
+        NL: /(\\r?\\n)+\s*/
         """ + common
 
         g2 = """
@@ -176,7 +176,7 @@ class TestReconstructor(TestCase):
         stmt: "குறிப்பு" var op var
         !op: ("+=" | "-=" | "*=" | "/=")
         var: WORD
-        NL: /(\\r?\\n)+\\s*/
+        NL: /(\\r?\\n)+\s*/
         """ + common
 
         code = """