nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: lark-parser/lark
Pull request #821 (2021-01-26 20:51:06)

Fix an SPPF cycle edge case (Issue #820)


Patch:
diff --git a/lark/parsers/earley_forest.py b/lark/parsers/earley_forest.py
index 03c4573..2317d73 100644
--- a/lark/parsers/earley_forest.py
+++ b/lark/parsers/earley_forest.py
@@ -507,7 +507,7 @@ class ForestToParseTree(ForestTransformer):
 
     def _check_cycle(self, node):
         if self._on_cycle_retreat:
-            if id(node) == id(self._cycle_node):
+            if id(node) == id(self._cycle_node) or id(node) in self._successful_visits:
                 self._cycle_node = None
                 self._on_cycle_retreat = False
                 return
@@ -541,16 +541,16 @@ class ForestToParseTree(ForestTransformer):
     def transform_symbol_node(self, node, data):
         if id(node) not in self._successful_visits:
             raise Discard()
-        self._successful_visits.remove(id(node))
         self._check_cycle(node)
+        self._successful_visits.remove(id(node))
         data = self._collapse_ambig(data)
         return self._call_ambig_func(node, data)
 
     def transform_intermediate_node(self, node, data):
         if id(node) not in self._successful_visits:
             raise Discard()
-        self._successful_visits.remove(id(node))
         self._check_cycle(node)
+        self._successful_visits.remove(id(node))
         if len(data) > 1:
             children = [self.tree_class('_inter', c) for c in data]
             return self.tree_class('_iambig', children)


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index daa7299..8500b01 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -775,6 +775,19 @@ def _make_full_earley_test(LEXER):
             tree = l.parse('')
             self.assertEqual(tree, Tree('start', []))
 
+        def test_cycle2(self):
+            grammar = """
+            start: _operation
+            _operation:  value
+            value: "b"
+                  | "a" value
+                  | _operation
+            """
+
+            l = Lark(grammar, ambiguity="explicit", lexer=LEXER)
+            tree = l.parse("ab")
+            self.assertEqual(tree, Tree('start', [Tree('value', [Tree('value', [])])]))
+
         def test_cycles(self):
             grammar = """
             a: b
Repository: lark-parser/lark
Pull request #827 (2021-02-05 14:11:06)

Fix for #826 + stubs + tests
Makes Transfomer_NonRecursive  call the Token callbacks. Also introduces Tests that all four Transformer variants do the same stuff.

The fix @mikeiovine proposed was exactly correct.


Patch:
diff --git a/lark-stubs/visitors.pyi b/lark-stubs/visitors.pyi
index 86f15d8..3a934ee 100644
--- a/lark-stubs/visitors.pyi
+++ b/lark-stubs/visitors.pyi
@@ -9,6 +9,7 @@ _R = TypeVar('_R')
 _FUNC = Callable[..., _T]
 _DECORATED = Union[_FUNC, type]
 
+
 class Transformer(ABC, Generic[_T]):
 
     def __init__(self, visit_tokens: bool = True) -> None:
@@ -38,6 +39,14 @@ class Transformer_InPlace(Transformer):
     pass
 
 
+class Transformer_NonRecursive(Transformer):
+    pass
+
+
+class Transformer_InPlaceRecursive(Transformer):
+    pass
+
+
 class VisitorBase:
     pass
 
@@ -73,10 +82,10 @@ _InterMethod = Callable[[Type[Interpreter], _T], _R]
 
 
 def v_args(
-    inline: bool = False,
-    meta: bool = False,
-    tree: bool = False,
-    wrapper: Callable = None
+        inline: bool = False,
+        meta: bool = False,
+        tree: bool = False,
+        wrapper: Callable = None
 ) -> Callable[[_DECORATED], _DECORATED]:
     ...
 
diff --git a/lark/visitors.py b/lark/visitors.py
index 7e3bae4..44f0c99 100644
--- a/lark/visitors.py
+++ b/lark/visitors.py
@@ -218,6 +218,8 @@ class Transformer_NonRecursive(Transformer):
                 else:
                     args = []
                 stack.append(self._call_userfunc(x, args))
+            elif self.__visit_tokens__ and isinstance(x, Token):
+                stack.append(self._call_userfunc_token(x))
             else:
                 stack.append(x)
 


Test patch:
diff --git a/tests/test_trees.py b/tests/test_trees.py
index 905ad5a..c7f9787 100644
--- a/tests/test_trees.py
+++ b/tests/test_trees.py
@@ -8,7 +8,8 @@ import functools
 
 from lark.tree import Tree
 from lark.lexer import Token
-from lark.visitors import Visitor, Visitor_Recursive, Transformer, Interpreter, visit_children_decor, v_args, Discard
+from lark.visitors import Visitor, Visitor_Recursive, Transformer, Interpreter, visit_children_decor, v_args, Discard, Transformer_InPlace, \
+    Transformer_InPlaceRecursive, Transformer_NonRecursive
 
 
 class TestTrees(TestCase):
@@ -232,6 +233,20 @@ class TestTrees(TestCase):
 
         x = MyTransformer().transform( t )
         self.assertEqual(x, t2)
+    
+    def test_transformer_variants(self):
+        tree = Tree('start', [Tree('add', [Token('N', '1'), Token('N', '2')]), Tree('add', [Token('N', '3'), Token('N', '4')])])
+        for base in (Transformer, Transformer_InPlace, Transformer_NonRecursive, Transformer_InPlaceRecursive):
+            class T(base):
+                def add(self, children):
+                    return sum(children)
+                
+                def N(self, token):
+                    return int(token)
+            
+            copied = copy.deepcopy(tree)
+            result = T().transform(copied)
+            self.assertEqual(result, Tree('start', [3, 7]))
 
 
 if __name__ == '__main__':
Repository: lark-parser/lark
Pull request #889 (2021-05-05 09:13:57)

Support 32bit unicode escape sequence
This PR adds 32bit unicode escape sequence and its test.

* 16bit unicode escape sequence `\uxxxx`: small `u` and 4 hexadecimal digits.
* 32bit unicode escape sequence `\Uxxxxxxxx`: capital `U` and 8 hexadecimal digits.

See 
https://docs.python.org/3.9/reference/lexical_analysis.html#string-and-bytes-literals


Patch:
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index 864407e..dcb4c81 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -414,7 +414,7 @@ def eval_escaping(s):
                 raise GrammarError("Literal ended unexpectedly (bad escaping): `%r`" % s)
             if n2 == '\\':
                 w += '\\\\'
-            elif n2 not in 'uxnftr':
+            elif n2 not in 'Uuxnftr':
                 w += '\\'
             w += n2
     w = w.replace('\\"', '"').replace("'", "\\'")


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 18b70fc..c543185 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -1074,6 +1074,14 @@ def _make_parser_test(LEXER, PARSER):
                         """)
             g.parse(u'\xa3\u0101\u00a3\u0203\n')
 
+        def test_unicode4(self):
+            g = _Lark(r"""start: UNIA UNIB UNIA UNIC
+                        UNIA: /\xa3/
+                        UNIB: "\U0010FFFF"
+                        UNIC: /\U00100000/ /\n/
+                        """)
+            g.parse(u'\xa3\U0010FFFF\u00a3\U00100000\n')
+
         def test_hex_escape(self):
             g = _Lark(r"""start: A B C
                           A: "\x01"
@@ -1088,6 +1096,13 @@ def _make_parser_test(LEXER, PARSER):
                           """)
             g.parse('abc')
 
+        @unittest.skipIf(sys.version_info < (3, 3), "re package did not support 32bit unicode escape sequence before Python 3.3")
+        def test_unicode_literal_range_escape2(self):
+            g = _Lark(r"""start: A+
+                          A: "\U0000FFFF".."\U00010002"
+                          """)
+            g.parse('\U0000FFFF\U00010000\U00010001\U00010002')
+
         def test_hex_literal_range_escape(self):
             g = _Lark(r"""start: A+
                           A: "\x01".."\x03"
Repository: lark-parser/lark
Pull request #980 (2021-08-29 21:11:47)

Sort Options inside a TerminalTree
Fixes the problem mentioned at the end of #970 and also inside of #977 (And I think it was mentioned in gitter in the last few days)

Note: This only fixes it for the common case, e.g. a direct listing of multiple options on after the other (e.g. `"a" | "ab"`). It probably doesn't fix it reliable when nesting is involved.

I think the sort key `(-x.max_width, -x.min_width, -len(x.value))` is correct/good enough, but I am not sure.


Patch:
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index 4a9360c..abcfce1 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -614,6 +614,10 @@ class TerminalTreeToPattern(Transformer_NonRecursive):
         if len(exps) == 1:
             return exps[0]
 
+        # Do a bit of sorting to make sure that the longest option is returned
+        # (Python's re module otherwise prefers just 'l' when given (l|ll) and both could match)
+        exps.sort(key=lambda x: (-x.max_width, -x.min_width, -len(x.value)))
+
         pattern = '(?:%s)' % ('|'.join(i.to_regexp() for i in exps))
         return _make_joined_pattern(pattern, {i.flags for i in exps})
 


Test patch:
diff --git a/tests/test_grammar.py b/tests/test_grammar.py
index 319d709..c771f2b 100644
--- a/tests/test_grammar.py
+++ b/tests/test_grammar.py
@@ -247,10 +247,8 @@ class TestGrammar(TestCase):
         self.assertRaises(UnexpectedInput, l.parse, u'A' * 8192)
 
     def test_large_terminal(self):
-        # TODO: The `reversed` below is required because otherwise the regex engine is happy
-        #       with just parsing 9 from the string 999 instead of consuming the longest
         g = "start: NUMBERS\n"
-        g += "NUMBERS: " + '|'.join('"%s"' % i for i in reversed(range(0, 1000)))
+        g += "NUMBERS: " + '|'.join('"%s"' % i for i in range(0, 1000))
 
         l = Lark(g, parser='lalr')
         for i in (0, 9, 99, 999):
Repository: lark-parser/lark
Pull request #990 (2021-09-06 13:16:16)

Grammar: Added support for line breaks, using \\


Patch:
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index 309826b..1ae832f 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -94,6 +94,7 @@ TERMINALS = {
     '_NL_OR': r'(\r?\n)+\s*\|',
     'WS': r'[ \t]+',
     'COMMENT': r'\s*//[^\n]*',
+    'BACKSLASH': r'\\[ ]*\n',
     '_TO': '->',
     '_IGNORE': r'%ignore',
     '_OVERRIDE': r'%override',
@@ -917,7 +918,7 @@ def _get_parser():
                  for r, _p, xs, o in rules for i, x in enumerate(xs)]
         callback = ParseTreeBuilder(rules, ST).create_callback()
         import re
-        lexer_conf = LexerConf(terminals, re, ['WS', 'COMMENT'])
+        lexer_conf = LexerConf(terminals, re, ['WS', 'COMMENT', 'BACKSLASH'])
         parser_conf = ParserConf(rules, callback, ['start'])
         lexer_conf.lexer_type = 'standard'
         parser_conf.parser_type = 'lalr'


Test patch:
diff --git a/tests/test_grammar.py b/tests/test_grammar.py
index c771f2b..c60d15b 100644
--- a/tests/test_grammar.py
+++ b/tests/test_grammar.py
@@ -270,6 +270,16 @@ class TestGrammar(TestCase):
             imports = list_grammar_imports('%import common.WS', [])
             assert len(imports) == 1 and imports[0].pkg_name == 'lark'
 
+    def test_line_breaks(self):
+        p = Lark(r"""start: "a" \
+                       "b"
+                """)
+        p.parse('ab')
+
+
+
+
+
 
 if __name__ == '__main__':
     main()
Repository: lark-parser/lark
Pull request #1080 (2022-01-15 18:59:28)

Bugfix: v_args failed to apply to class under standalone parser
Solves issue #1059


Patch:
diff --git a/lark/visitors.py b/lark/visitors.py
index 262562f..fd49ca0 100644
--- a/lark/visitors.py
+++ b/lark/visitors.py
@@ -1,6 +1,5 @@
 from typing import TypeVar, Tuple, List, Callable, Generic, Type, Union, Optional, Any
 from abc import ABC
-from functools import wraps, update_wrapper
 
 from .utils import combine_alternatives
 from .tree import Tree
@@ -8,6 +7,7 @@ from .exceptions import VisitError, GrammarError
 from .lexer import Token
 
 ###{standalone
+from functools import wraps, update_wrapper
 from inspect import getmembers, getmro
 
 _T = TypeVar('_T')


Test patch:
diff --git a/tests/test_tools.py b/tests/test_tools.py
index 80e69b0..deef929 100644
--- a/tests/test_tools.py
+++ b/tests/test_tools.py
@@ -64,12 +64,15 @@ class TestStandalone(TestCase):
         l = _Lark()
         x = l.parse('ABAB')
 
+        _v_args = context['v_args']
+        @_v_args(inline=True)
         class T(context['Transformer']):
-            def a(self, items):
+            def a(self):
                 return 'a'
-            def b(self, items):
+            def b(self):
                 return 'b'
-            start = list
+
+            start = _v_args(inline=False)(list)
 
         x = T().transform(x)
         self.assertEqual(x, ['a', 'b'])
Repository: lark-parser/lark
Pull request #1198 (2022-10-07 18:05:17)

Fix for issue #1173


Patch:
diff --git a/lark/load_grammar.py b/lark/load_grammar.py
index 1f13b5d..d4f553c 100644
--- a/lark/load_grammar.py
+++ b/lark/load_grammar.py
@@ -1050,6 +1050,8 @@ def _make_rule_tuple(modifiers_tree, name, params, priority_tree, expansions):
     if modifiers_tree.children:
         m ,= modifiers_tree.children
         expand1 = '?' in m
+        if expand1 and name.startswith('_'):
+            raise GrammarError("Inlined rules (_rule) cannot use the ?rule modifier.")
         keep_all_tokens = '!' in m
     else:
         keep_all_tokens = False


Test patch:
diff --git a/tests/test_grammar.py b/tests/test_grammar.py
index 7816189..e388cb7 100644
--- a/tests/test_grammar.py
+++ b/tests/test_grammar.py
@@ -281,6 +281,14 @@ class TestGrammar(TestCase):
             imports = list_grammar_imports('%import common.WS', [])
             assert len(imports) == 1 and imports[0].pkg_name == 'lark'
 
+    def test_inline_with_expand_single(self):
+        grammar = r"""
+        start: _a
+        !?_a: "A"
+        """
+        self.assertRaises(GrammarError, Lark, grammar)
+
+
     def test_line_breaks(self):
         p = Lark(r"""start: "a" \
                        "b"
Repository: lark-parser/lark
Pull request #1347 (2023-09-28 11:38:46)

Fix 1345 attempt two
Alternative to #1346


Patch:
diff --git a/lark/parsers/lalr_interactive_parser.py b/lark/parsers/lalr_interactive_parser.py
index 7ee4001..2a30d8e 100644
--- a/lark/parsers/lalr_interactive_parser.py
+++ b/lark/parsers/lalr_interactive_parser.py
@@ -102,9 +102,14 @@ class InteractiveParser:
     def accepts(self):
         """Returns the set of possible tokens that will advance the parser into a new valid state."""
         accepts = set()
+        conf_no_callbacks = copy(self.parser_state.parse_conf)
+        # We don't want to call callbacks here since those might have arbitrary side effects
+        # and are unnecessarily slow.
+        conf_no_callbacks.callbacks = {}
         for t in self.choices():
             if t.isupper(): # is terminal?
                 new_cursor = copy(self)
+                new_cursor.parser_state.parse_conf = conf_no_callbacks
                 try:
                     new_cursor.feed_token(self.lexer_thread._Token(t, ''))
                 except UnexpectedToken:
diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py
index c330a4d..5a37dc5 100644
--- a/lark/parsers/lalr_parser.py
+++ b/lark/parsers/lalr_parser.py
@@ -147,7 +147,7 @@ class ParserState:
                 else:
                     s = []
 
-                value = callbacks[rule](s)
+                value = callbacks[rule](s) if callbacks else s
 
                 _action, new_state = states[state_stack[-1]][rule.origin.name]
                 assert _action is Shift


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index ccddf63..839e16a 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -2595,6 +2595,28 @@ def _make_parser_test(LEXER, PARSER):
             assert next(i) == 'a'
             assert next(i) == 'b'
 
+        @unittest.skipIf(PARSER!='lalr', "interactive_parser is only implemented for LALR at the moment")
+        def test_interactive_treeless_transformer(self):
+            grammar = r"""
+                start: SYM+
+
+                SYM: "a" | "b"
+            """
+
+            class SYMTransformer(lark.visitors.Transformer):
+                def SYM(self, token):
+                    return {"a": 1, "b": 2}[str(token)]
+
+            parser = _Lark(grammar, transformer=SYMTransformer())
+            res = parser.parse("aba")
+            self.assertEqual(res.children, [1, 2, 1])
+            ip = parser.parse_interactive("aba")
+            ip.exhaust_lexer()
+            # Previously `accepts` would call `SYMTransformer.SYM` with `Token('SYM', '')`, which would cause an error.
+            self.assertEqual(ip.accepts(), {"$END", "SYM"})
+            res = ip.feed_eof()
+            self.assertEqual(res.children, [1, 2, 1])
+
         @unittest.skipIf(PARSER!='lalr', "Tree-less mode is only supported in lalr")
         def test_default_in_treeless_mode(self):
             grammar = r"""
Repository: lark-parser/lark
Pull request #1444 (2024-08-13 17:39:16)

Bugfix: Earley now respects ambiguity='resolve' again. Bug was introd…
Reported in #1443 

The bug was introduced in a previous bugfix.


Patch:
diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py
index 8deaa2f..d173129 100644
--- a/lark/parsers/earley.py
+++ b/lark/parsers/earley.py
@@ -306,7 +306,7 @@ class Parser:
             transformer = ForestToParseTree(self.Tree, self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor(), self.resolve_ambiguity, use_cache)
             solutions = [transformer.transform(s) for s in solutions]
 
-            if len(solutions) > 1:
+            if len(solutions) > 1 and not self.resolve_ambiguity:
                 t: Tree = self.Tree('_ambig', solutions)
                 t.expand_kids_by_data('_ambig')     # solutions may themselves be _ambig nodes
                 return t


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 5a0d405..59e9a71 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -841,6 +841,11 @@ def _make_full_earley_test(LEXER):
             )
             self.assertEqual(tree, expected)
 
+            l = Lark(grammar, ambiguity='resolve', lexer=LEXER)
+            tree = l.parse('x')
+            assert tree == Tree('start', ['x'])
+
+
         def test_cycle(self):
             grammar = """
             start: start?
Repository: lark-parser/lark
Pull request #1460 (2024-09-01 09:17:52)

Better error in Lark.parse when using on_error when parser!=lalr (issue #1311)
Not much to review, but I'll still leave it up for comments for a while


Patch:
diff --git a/lark/lark.py b/lark/lark.py
index 7ae1f24..0bec71b 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -652,6 +652,8 @@ class Lark(Serialize):
                 For convenience, these sub-exceptions also inherit from ``ParserError`` and ``LexerError``.
 
         """
+        if on_error is not None and self.options.parser != 'lalr':
+            raise NotImplementedError("The on_error option is only implemented for the LALR(1) parser.")
         return self.parser.parse(text, start=start, on_error=on_error)
 
 


Test patch:
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 9829071..36a9cc6 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -2584,6 +2584,11 @@ def _make_parser_test(LEXER, PARSER):
             s = "[0 1, 2,@, 3,,, 4, 5 6 ]$"
             tree = g.parse(s, on_error=ignore_errors)
 
+        @unittest.skipIf(PARSER == 'lalr', "test on_error only works with lalr")
+        def test_on_error_without_lalr(self):
+            p = _Lark(r"""start: "A" """)
+            self.assertRaises(NotImplementedError, p.parse, "", on_error=print)
+
         @unittest.skipIf(PARSER != 'lalr', "interactive_parser error handling only works with LALR for now")
         def test_iter_parse(self):
             ab_grammar = '!start: "a"* "b"*'