Repository: lark-parser/lark

Issue:
Lark cache fails when custom lexers are used
**Describe the bug**

In parser_frontends.py, the passed lexer is wrapped using functools.partial: 
```
return partial(LALR_CustomLexer, lexer)
```

but this results in a functools.partial object, not a lexer object, so when deserialize is called later, it fails with
```
AttributeError: 'functools.partial' object has no attribute 'deserialize'
--> 338         self.parser = self.parser_class.deserialize(data['parser'], memo, self._callbacks, self.options.postlex, self.re)
```



**To Reproduce**
I've modified the custom lexer example

```
from lark import Lark, Transformer, v_args
from lark.lexer import Lexer, Token

class TypeLexer(Lexer):
    def __init__(self, lexer_conf, re_=None):
        pass

    def lex(self, data):
        for obj in data:
            if isinstance(obj, int):
                yield Token('INT', obj)
            elif isinstance(obj, (type(''), type(u''))):
                yield Token('STR', obj)
            else:
                raise TypeError(obj)

grammar = """
        start: data_item+
        data_item: STR INT*
        %declare STR INT
        """
parser = Lark(grammar, parser='lalr', lexer=TypeLexer, cache=True)

# Fails
parser = Lark(grammar, parser='lalr', lexer=TypeLexer, cache=True)
```


Discussion:
erezsh: The patch looks good.

But for some reason I was sure we already merged something like this.. I wonder why
MegaIng: I also thought the same, but apparently not.
erezsh: @adefazio would it be okay if I asked you to rebase this into two commits? Things like "newline" kind of clutter the log.

I can do it myself if you prefer, but I think that might mess with github's references.
adefazio: > @adefazio would it be okay if I asked you to rebase this into two commits? Things like "newline" kind of clutter the log.
> 
> I can do it myself if you prefer, but I think that might mess with github's references.

Ok, I'll do that.

Patch:
diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py
index 08f4756..c05f235 100644
--- a/lark/parser_frontends.py
+++ b/lark/parser_frontends.py
@@ -1,5 +1,3 @@
-from functools import partial
-
 from .utils import get_regexp_width, Serialize
 from .parsers.grammar_analysis import GrammarAnalyzer
 from .lexer import TraditionalLexer, ContextualLexer, Lexer, Token
@@ -20,7 +18,14 @@ def get_frontend(parser, lexer):
         elif lexer == 'contextual':
             return LALR_ContextualLexer
         elif issubclass(lexer, Lexer):
-            return partial(LALR_CustomLexer, lexer)
+            class LALR_CustomLexerWrapper(LALR_CustomLexer):
+                def __init__(self, lexer_conf, parser_conf, options=None):
+                    super(LALR_CustomLexerWrapper, self).__init__(
+                        lexer, lexer_conf, parser_conf, options=options)
+                def init_lexer(self):
+                    self.lexer = lexer(self.lexer_conf)
+
+            return LALR_CustomLexerWrapper
         else:
             raise ValueError('Unknown lexer: %s' % lexer)
     elif parser=='earley':


Test patch:
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 9436081..ca4d781 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -4,6 +4,7 @@ import sys
 from unittest import TestCase, main
 
 from lark import Lark, Tree
+from lark.lexer import Lexer, Token
 import lark.lark as lark_module
 
 try:
@@ -38,6 +39,15 @@ class MockFS:
         return name in self.files
 
 
+class CustomLexer(Lexer):
+    def __init__(self, lexer_conf):
+        pass
+
+    def lex(self, data):
+        for obj in data:
+            yield Token('A', obj)
+
+
 class TestCache(TestCase):
     def setUp(self):
         pass
@@ -70,6 +80,12 @@ class TestCache(TestCase):
             parser = Lark(g, parser='lalr', cache=True)
             assert parser.parse('a') == Tree('start', [])
 
+            # Test with custom lexer
+            mock_fs.files = {}
+            parser = Lark(g, parser='lalr', lexer=CustomLexer, cache=True)
+            parser = Lark(g, parser='lalr', lexer=CustomLexer, cache=True)
+            assert len(mock_fs.files) == 1
+            assert parser.parse('a') == Tree('start', [])
         finally:
             lark_module.FS = fs