Repository: lark-parser/lark

Issue:
Dynamic Earley: Incorrect value for SymbolNode.end
Consider the following:

```python
grammar = r"""
start: "ABC"
"""

print(Lark(grammar, ambiguity="forest").parse("ABC"))
```

Output: `(start, 0, 2, -inf)`

The end index is 2 which suggests that the substring "AB" was parsed. However, we know that the full string was parsed, so the end index should be 3.

Note that the issue does not occur if we use a rule instead:

```python
grammar = r"""
start: abc 
abc: "ABC"
"""

print(Lark(grammar, ambiguity="forest").parse("ABC"))
```

Output: `(start, 0, 3, -inf)`

I will open a PR with a fix.


Discussion:

Patch:
diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py
index 301f4d1..cdc613a 100644
--- a/lark/parsers/earley_forest.py
+++ b/lark/parsers/earley_forest.py
@@ -38,15 +38,15 @@ class SymbolNode(ForestNode):
 
     Parameters:
         s: A Symbol, or a tuple of (rule, ptr) for an intermediate node.
-        start: The index of the start of the substring matched by this symbol (inclusive).
-        end: The index of the end of the substring matched by this symbol (exclusive).
+        start: For dynamic lexers, the index of the start of the substring matched by this symbol (inclusive).
+        end: For dynamic lexers, the index of the end of the substring matched by this symbol (exclusive).
 
     Properties:
         is_intermediate: True if this node is an intermediate node.
         priority: The priority of the node's symbol.
     """
     Set: Type[AbstractSet] = set   # Overridden by StableSymbolNode
-    __slots__ = ('s', 'start', 'end', '_children', 'paths', 'paths_loaded', 'priority', 'is_intermediate', '_hash')
+    __slots__ = ('s', 'start', 'end', '_children', 'paths', 'paths_loaded', 'priority', 'is_intermediate')
     def __init__(self, s, start, end):
         self.s = s
         self.start = start
@@ -59,7 +59,6 @@ class SymbolNode(ForestNode):
         #   unlike None or float('NaN'), and sorts appropriately.
         self.priority = float('-inf')
         self.is_intermediate = isinstance(s, tuple)
-        self._hash = hash((self.s, self.start, self.end))
 
     def add_family(self, lr0, rule, start, left, right):
         self._children.add(PackedNode(self, lr0, rule, start, left, right))
@@ -93,14 +92,6 @@ class SymbolNode(ForestNode):
     def __iter__(self):
         return iter(self._children)
 
-    def __eq__(self, other):
-        if not isinstance(other, SymbolNode):
-            return False
-        return self is other or (type(self.s) == type(other.s) and self.s == other.s and self.start == other.start and self.end is other.end)
-
-    def __hash__(self):
-        return self._hash
-
     def __repr__(self):
         if self.is_intermediate:
             rule = self.s[0]
diff --git a/lark/parsers/xearley.py b/lark/parsers/xearley.py
index d9748df..a0f43ac 100644
--- a/lark/parsers/xearley.py
+++ b/lark/parsers/xearley.py
@@ -104,7 +104,7 @@ class Parser(BaseParser):
                     token.end_pos = i + 1
 
                     new_item = item.advance()
-                    label = (new_item.s, new_item.start, i)
+                    label = (new_item.s, new_item.start, i + 1)
                     token_node = TokenNode(token, terminals[token.type])
                     new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token_node)


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index d94424a..ee5d5a0 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -933,6 +933,27 @@ def _make_full_earley_test(LEXER):
             tree = l.parse('');
             self.assertEqual(tree, Tree('a', [Tree('x', [Tree('b', [])])]))
 
+        @unittest.skipIf(LEXER=='basic', "start/end values work differently for the basic lexer")
+        def test_symbol_node_start_end_dynamic_lexer(self):
+            grammar = """
+            start: "ABC"
+            """
+
+            l = Lark(grammar, ambiguity='forest', lexer=LEXER)
+            node = l.parse('ABC')
+            self.assertEqual(node.start, 0)
+            self.assertEqual(node.end, 3)
+
+            grammar2 = """
+            start: abc
+            abc: "ABC"
+            """
+
+            l = Lark(grammar2, ambiguity='forest', lexer=LEXER)
+            node = l.parse('ABC')
+            self.assertEqual(node.start, 0)
+            self.assertEqual(node.end, 3)
+
 
     _NAME = "TestFullEarley" + LEXER.capitalize()
     _TestFullEarley.__name__ = _NAME