Repository: lark-parser/lark

Issue:
Transformer is not applied for tokens in standalone parser
**Describe the bug**
Standalone parser doesn't invoke methods on tokens defined in supplied transformer

**To Reproduce**
Given:
- grammar `example.lark`:
```
start: IDENTIFIER "(" text ")"
IDENTIFIER: /[A-Za-z_][A-Za-z0-9_]*/
text: IDENTIFIER
```
- standalone parser generated with: `python3 -m lark.tools.standalone example.lark > example.py`

Then:
- this program:
```python
import lark

class DowncaseIdentifiers(lark.Transformer):
    def IDENTIFIER(self, token):
        return lark.Token("IDENTIFIER", token.lower())

    def text(self, children):
        return lark.Tree("text", [self.IDENTIFIER(children[0])])

parser = lark.Lark.open(
    grammar_filename="example.lark", transformer=DowncaseIdentifiers(), parser="lalr"
)
to_parse = "FOO(BAR)"
result = parser.parse(to_parse)
print(result.pretty())
```
produces:
```
start
  foo
  text  bar
```
- this program:
```python
import example

class DowncaseIdentifiers(example.Transformer):
    def IDENTIFIER(self, token):
        return example.Token("IDENTIFIER", token.lower())

    def text(self, children):
        return example.Tree("text", [self.IDENTIFIER(children[0])])

parser = example.Lark_StandAlone(transformer=DowncaseIdentifiers())
to_parse = "FOO(BAR)"
result = parser.parse(to_parse)
print(result.pretty())
```
produces:
```
start
  FOO
  text  bar
```

Expected:
Both programs produce the same output, namely:
```
start
  foo
  text  bar
```

Discussion:

Patch:
diff --git a/lark/common.py b/lark/common.py
index cc8c73c..714399a 100644
--- a/lark/common.py
+++ b/lark/common.py
@@ -17,9 +17,6 @@ class LexerConf(Serialize):
         self.skip_validation = skip_validation
         self.use_bytes = use_bytes
 
-    def _deserialize(self):
-        self.callbacks = {} # TODO
-
 ###}
 
 class ParserConf:
diff --git a/lark/lark.py b/lark/lark.py
index daab45b..3ed96d7 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -344,7 +344,14 @@ class Lark(Serialize):
         self.rules = [Rule.deserialize(r, memo) for r in data['rules']]
         self.source = '<deserialized>'
         self._prepare_callbacks()
-        self.parser = self.parser_class.deserialize(data['parser'], memo, self._callbacks, self.options.postlex, re_module)
+        self.parser = self.parser_class.deserialize(
+            data['parser'],
+            memo,
+            self._callbacks,
+            self.options.postlex,
+            self.options.transformer,
+            re_module
+        )
         return self
 
     @classmethod
diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py
index 33ad9bc..a45bf9c 100644
--- a/lark/parser_frontends.py
+++ b/lark/parser_frontends.py
@@ -1,6 +1,6 @@
 from .utils import get_regexp_width, Serialize
 from .parsers.grammar_analysis import GrammarAnalyzer
-from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token
+from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token, TerminalDef
 from .parsers import earley, xearley, cyk
 from .parsers.lalr_parser import LALR_Parser
 from .grammar import Rule
@@ -58,6 +58,16 @@ class _ParserFrontend(Serialize):
         return self.parser.parse(input, start, *args)
 
 
+def _recreate_lexer_callbacks(memo, transformer):
+    result = {}
+    terminals = [item for item in memo.values() if isinstance(item, TerminalDef)]
+    for terminal in terminals:
+        callback = getattr(transformer, terminal.name, None)
+        if callback is not None:
+            result[terminal.name] = callback
+    return result
+
+
 class WithLexer(_ParserFrontend):
     lexer = None
     parser = None
@@ -73,10 +83,11 @@ class WithLexer(_ParserFrontend):
         self.postlex = lexer_conf.postlex
 
     @classmethod
-    def deserialize(cls, data, memo, callbacks, postlex, re_module):
+    def deserialize(cls, data, memo, callbacks, postlex, transformer, re_module):
         inst = super(WithLexer, cls).deserialize(data, memo)
         inst.postlex = postlex
         inst.parser = LALR_Parser.deserialize(inst.parser, memo, callbacks)
+        inst.lexer_conf.callbacks = _recreate_lexer_callbacks(memo, transformer)
         inst.lexer_conf.re_module = re_module
         inst.lexer_conf.skip_validation=True
         inst.init_lexer()
@@ -229,4 +240,3 @@ class CYK(WithLexer):
 
     def _apply_callback(self, tree):
         return self.callbacks[tree.rule](tree.children)
-


Test patch:
diff --git a/tests/test_tools.py b/tests/test_tools.py
index 1e0d78e..e691237 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -106,6 +106,33 @@ class TestStandalone(TestCase):
         x = l.parse('(\n)\n')
         self.assertEqual(x, Tree('start', []))
 
+    def test_transformer(self):
+        grammar = r"""
+            start: some_rule "(" SOME_TERMINAL ")"
+            some_rule: SOME_TERMINAL
+            SOME_TERMINAL: /[A-Za-z_][A-Za-z0-9_]*/
+        """
+        context = self._create_standalone(grammar)
+        _Lark = context["Lark_StandAlone"]
+
+        _Token = context["Token"]
+        _Tree = context["Tree"]
+
+        class MyTransformer(context["Transformer"]):
+            def SOME_TERMINAL(self, token):
+                return _Token("SOME_TERMINAL", "token is transformed")
+
+            def some_rule(self, children):
+                return _Tree("rule_is_transformed", [])
+
+        parser = _Lark(transformer=MyTransformer())
+        self.assertEqual(
+            parser.parse("FOO(BAR)"),
+            _Tree("start", [
+                _Tree("rule_is_transformed", []),
+                _Token("SOME_TERMINAL", "token is transformed")
+            ])
+        )
 
 
 if __name__ == '__main__':