Repository: lark-parser/lark

Issue:
ParserPuppet.copy does not copy LexerState
**Describe the bug**

`ParserPuppet.copy` does not copy the lexer state. Instead it makes a shallow copy of `LexerThread`.

**To Reproduce**

```python3
# make a copy
puppet_copy = puppet.copy()

# advance LineCounter of copy
puppet_copy.lexer_state.state.line_ctr.feed('foo')

# assertion fails
assert puppet.lexer_state.state.line_ctr.char_pos != puppet_copy.lexer_state.state.line_ctr.char_pos
```

This should be simple to fix. The reason for the bug is that the member `lexer_state` of `ParserPuppet` is in fact not a `LexerState` but a `LexerThread` (see [here](https://github.com/lark-parser/lark/blob/1d61de4f94b4af441b2f32c790db4290348d7a93/lark/parsers/lalr_parser.py#L149)), which does not have a custom copy function. So `copy(self.lexer_state)` would only be a shallow copy. I think we can just add a `__copy__` method to `LexerThread` ([here](https://github.com/lark-parser/lark/blob/1d61de4f94b4af441b2f32c790db4290348d7a93/lark/lexer.py#L459)) to fix this, namely
```python3
    def __copy__(self):
        return type(self)(
            self.lexer,
            copy(self.state)
        )
```

Discussion:

Patch:
diff --git a/lark/lexer.py b/lark/lexer.py
index 5a8d21a..64dc33e 100644
--- a/lark/lexer.py
+++ b/lark/lexer.py
@@ -186,6 +186,13 @@ class LineCounter:
         self.column = 1
         self.line_start_pos = 0
 
+    def __eq__(self, other):
+        if not isinstance(other, LineCounter):
+            return NotImplemented
+        return (self.newline_char == other.newline_char and self.char_pos == other.char_pos
+            and self.line == other.line and self.column == other.column
+            and self.line_start_pos == other.line_start_pos)
+
     def feed(self, token, test_newline=True):
         """Consume a token and calculate the new line & column.
 
@@ -397,7 +404,7 @@ class TraditionalLexer(Lexer):
         raise EOFError(self)
 
 
-class LexerState:
+class LexerState(object):
     __slots__ = 'text', 'line_ctr', 'last_token'
 
     def __init__(self, text, line_ctr, last_token=None):
@@ -405,6 +412,13 @@ class LexerState:
         self.line_ctr = line_ctr
         self.last_token = last_token
 
+    def __eq__(self, other):
+        if not isinstance(other, LexerState):
+            return NotImplemented
+
+        return (self.text == other.text and self.line_ctr == other.line_ctr
+            and self.last_token == other.last_token)
+
     def __copy__(self):
         return type(self)(self.text, copy(self.line_ctr), self.last_token)
 
@@ -456,7 +470,7 @@ class ContextualLexer(Lexer):
             except UnexpectedCharacters:
                 raise e  # Raise the original UnexpectedCharacters. The root lexer raises it with the wrong expected set.
 
-class LexerThread:
+class LexerThread(object):
     """A thread that ties a lexer instance and a lexer state, to be used by the parser"""
 
     def __init__(self, lexer, text):
@@ -465,4 +479,10 @@ class LexerThread:
 
     def lex(self, parser_state):
         return self.lexer.lex(self.state, parser_state)
+
+    def __copy__(self):
+        copied = object.__new__(LexerThread)
+        copied.lexer = self.lexer
+        copied.state = copy(self.state)
+        return copied
 ###}
diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py
index 6271bd5..6fc76ea 100644
--- a/lark/parsers/lalr_parser.py
+++ b/lark/parsers/lalr_parser.py
@@ -94,8 +94,8 @@ class ParserState(object):
     # Necessary for match_examples() to work
     def __eq__(self, other):
         if not isinstance(other, ParserState):
-            return False
-        return self.position == other.position
+            return NotImplemented
+        return len(self.state_stack) == len(other.state_stack) and self.position == other.position
 
     def __copy__(self):
         return type(self)(


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index ef91a92..3d18b20 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -2408,9 +2408,18 @@ def _make_parser_test(LEXER, PARSER):
                     # Skip comma
                     return True
                 elif e.token.type == 'SIGNED_NUMBER':
+                    # Make a copy and ensure it is properly made
+                    puppet_copy = e.puppet.copy()
+                    assert puppet_copy.parser_state == e.puppet.parser_state
+                    assert puppet_copy.lexer_state.state == e.puppet.lexer_state.state
+                    assert puppet_copy.parser_state is not e.puppet.parser_state
+                    assert puppet_copy.lexer_state.state is not e.puppet.lexer_state.state
+                    assert puppet_copy.lexer_state.state.line_ctr is not e.puppet.lexer_state.state.line_ctr
+
                     # Try to feed a comma and retry the number
                     e.puppet.feed_token(Token('COMMA', ','))
                     e.puppet.feed_token(e.token)
+
                     return True
 
                 # Unhandled error. Will stop parse and raise exception