Repository: lark-parser/lark

Issue:
python 3.14.0a1 - test_partial and test_smart_decorator are failing 
**Describe the bug**

Fedora is starting to rebuild packages with upcoming python 3.14. Some tests are failing.

~~~~
======================================================================
ERROR: test_partial (tests.test_trees.TestTrees.test_partial)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 122, in _call_userfunc
    return f.visit_wrapper(f, tree.data, children, tree.meta)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 504, in _vargs_inline
    return f(*children)
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 482, in __call__
    return self.base_func(*args, **kwargs)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
TypeError: TestTrees.test_partial.<locals>.test() got multiple values for argument 'postfix'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/tests/test_trees.py", line 361, in test_partial
    res = T().transform(tree)
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 161, in transform
    res = list(self._transform_children([tree]))
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 146, in _transform_children
    res = self._transform_tree(c)
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 156, in _transform_tree
    children = list(self._transform_children(tree.children))
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 146, in _transform_children
    res = self._transform_tree(c)
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 157, in _transform_tree
    return self._call_userfunc(tree, children)
           ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 128, in _call_userfunc
    raise VisitError(tree.data, tree, e)
lark.exceptions.VisitError: Error trying to process rule "a":

TestTrees.test_partial.<locals>.test() got multiple values for argument 'postfix'

======================================================================
ERROR: test_smart_decorator (tests.test_trees.TestTrees.test_smart_decorator)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 122, in _call_userfunc
    return f.visit_wrapper(f, tree.data, children, tree.meta)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 504, in _vargs_inline
    return f(*children)
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 482, in __call__
    return self.base_func(*args, **kwargs)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
TypeError: reduce() arg 2 must support iteration

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/tests/test_trees.py", line 293, in test_smart_decorator
    result = test_instance.transform(Tree(method_name, children))
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 161, in transform
    res = list(self._transform_children([tree]))
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 146, in _transform_children
    res = self._transform_tree(c)
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 157, in _transform_tree
    return self._call_userfunc(tree, children)
           ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^
  File "/builddir/build/BUILD/python-lark-1.2.2-build/lark-1.2.2/lark/visitors.py", line 128, in _call_userfunc
    raise VisitError(tree.data, tree, e)
lark.exceptions.VisitError: Error trying to process rule "partial_reduce_mul":

reduce() arg 2 must support iteration

----------------------------------------------------------------------
Ran 1115 tests in 8.042s

FAILED (errors=2, skipped=131)
~~~~

Discussion:
@MegaIng: This means that `partial` change it's interface in some way in case someone else wants to dig up what exactly it is in the python changelog.

Since Python3.14 is still in the alpha phase, this might also be a bug/unintended breaking change on their side.@hroncok: https://github.com/python/cpython/pull/121089 Make the [functools.partial](https://docs.python.org/3.14/library/functools.html#functools.partial) object a method descriptor.

This sounds possibly related.@MegaIng: Yeah, also found that.

So CPython is performing a breaking change with no deprecation warnings and less than 1 version `FutureWarning` :-/

Solution patch:
diff --git a/docs/conf.py b/docs/conf.py
index aacf8bb..59c2ac6 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -76,7 +76,7 @@ release = ''
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
diff --git a/docs/grammar.md b/docs/grammar.md
index 14179d0..b136bd2 100644
--- a/docs/grammar.md
+++ b/docs/grammar.md
@@ -239,6 +239,9 @@ Each item is one of:
 * `item ~ n` - Exactly *n* instances of item
 * `item ~ n..m` - Between *n* to *m* instances of item (not recommended for wide ranges, due to performance issues)
 
+Note that all operators, including `~ n`, apply directly to the item on their left.
+For instance `a ~ 2 b ~ 3` is parsed as `(a ~ 2) (b ~ 3)`.
+
 **Examples:**
 ```perl
 hello_world: "hello" "world"
diff --git a/docs/how_to_develop.md b/docs/how_to_develop.md
index cf95fed..a174c2e 100644
--- a/docs/how_to_develop.md
+++ b/docs/how_to_develop.md
@@ -65,3 +65,15 @@ Another way to run the tests is using setup.py:
 ```bash
 python setup.py test
 ```
+
+## Building the Documentation
+
+To build the documentation:
+
+```sh
+cd docs/
+pip install -r requirements.txt
+make html
+```
+
+To review the result, open the built HTML files under `_build/html/` in your browser.
diff --git a/docs/recipes.md b/docs/recipes.md
index 8101e37..b1e7a8b 100644
--- a/docs/recipes.md
+++ b/docs/recipes.md
@@ -79,6 +79,7 @@ Prints out:
 
 *Note: We don't have to return a token, because comments are ignored*
 
+
 ## CollapseAmbiguities
 
 Parsing ambiguous texts with earley and `ambiguity='explicit'` produces a single tree with `_ambig` nodes to mark where the ambiguity occurred.
@@ -193,3 +194,13 @@ def parse_with_progress(parser: Lark, text: str, start=None):
 ```
 
 Keep in mind that this implementation relies on the `InteractiveParser` and, therefore, only works with the `LALR(1)` parser, and not `Earley`.
+
+
+## Parsing a Language with Significant Indentation
+
+If your grammar needs to support significant indentation (e.g. Python, YAML), you will need to use
+the `Indenter` class. Take a look at the [indented tree example][indent] as well as the
+[Python grammar][python] for inspiration.
+
+[indent]: examples/indented_tree.html
+[python]: https://github.com/lark-parser/lark/blob/master/lark/grammars/python.lark
diff --git a/examples/indented_tree.py b/examples/indented_tree.py
index 5ac928c..a43733e 100644
--- a/examples/indented_tree.py
+++ b/examples/indented_tree.py
@@ -3,28 +3,34 @@ Parsing Indentation
 ===================
 
 A demonstration of parsing indentation (“whitespace significant” language)
-and the usage of the Indenter class.
+and the usage of the ``Indenter`` class.
 
 Since indentation is context-sensitive, a postlex stage is introduced to
-manufacture INDENT/DEDENT tokens.
+manufacture ``INDENT``/``DEDENT`` tokens.
 
-It is crucial for the indenter that the NL_type matches
-the spaces (and tabs) after the newline.
+It is crucial for the indenter that the ``NL_type`` matches the spaces (and
+tabs) after the newline.
+
+If your whitespace-significant grammar supports comments, then ``NL_type``
+must match those comments too. Otherwise, comments that appear in the middle
+of a line will `confuse Lark`_.
+
+.. _`confuse Lark`: https://github.com/lark-parser/lark/issues/863
 """
 from lark import Lark
 from lark.indenter import Indenter
 
 tree_grammar = r"""
-    ?start: _NL* tree
-
-    tree: NAME _NL [_INDENT tree+ _DEDENT]
-
     %import common.CNAME -> NAME
     %import common.WS_INLINE
-    %declare _INDENT _DEDENT
+    %import common.SH_COMMENT
     %ignore WS_INLINE
+    %ignore SH_COMMENT
+    %declare _INDENT _DEDENT
 
-    _NL: /(\r?\n[\t ]*)+/
+    ?start: _NL* tree
+    tree: NAME _NL [_INDENT tree+ _DEDENT]
+    _NL: (/\r?\n[\t ]*/ | SH_COMMENT)+
 """
 
 class TreeIndenter(Indenter):
@@ -39,6 +45,7 @@ parser = Lark(tree_grammar, parser='lalr', postlex=TreeIndenter())
 
 test_tree = """
 a
+    # check this comment out
     b
     c
         d
diff --git a/lark/grammar.py b/lark/grammar.py
index 016496b..3aa8e3a 100644
--- a/lark/grammar.py
+++ b/lark/grammar.py
@@ -40,7 +40,7 @@ class Terminal(Symbol):
 
     is_term: ClassVar[bool] = True
 
-    def __init__(self, name, filter_out=False):
+    def __init__(self, name: str, filter_out: bool = False) -> None:
         self.name = name
         self.filter_out = filter_out
 
diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py
index e3a4171..1acad4d 100644
--- a/lark/parse_tree_builder.py
+++ b/lark/parse_tree_builder.py
@@ -320,7 +320,7 @@ def inplace_transformer(func):
 
 def apply_visit_wrapper(func, name, wrapper):
     if wrapper is _vargs_meta or wrapper is _vargs_meta_inline:
-        raise NotImplementedError("Meta args not supported for internal transformer")
+        raise NotImplementedError("Meta args not supported for internal transformer; use YourTransformer().transform(parser.parse()) instead")
 
     @wraps(func)
     def f(children):
diff --git a/lark/tree.py b/lark/tree.py
index 76f8738..9dccadd 100644
--- a/lark/tree.py
+++ b/lark/tree.py
@@ -3,8 +3,10 @@ from copy import deepcopy
 
 from typing import List, Callable, Iterator, Union, Optional, Generic, TypeVar, TYPE_CHECKING
 
+from .lexer import Token
+
 if TYPE_CHECKING:
-    from .lexer import TerminalDef, Token
+    from .lexer import TerminalDef
     try:
         import rich
     except ImportError:
@@ -171,6 +173,16 @@ class Tree(Generic[_Leaf_T]):
 
 ###}
 
+    def find_token(self, token_type: str) -> Iterator[_Leaf_T]:
+        """Returns all tokens whose type equals the given token_type.
+
+        This is a recursive function that will find tokens in all the subtrees.
+
+        Example:
+            >>> term_tokens = tree.find_token('TERM')
+        """
+        return self.scan_values(lambda v: isinstance(v, Token) and v.type == token_type)
+
     def expand_kids_by_data(self, *data_values):
         """Expand (inline) children with any of the given data values. Returns True if anything changed"""
         changed = False
diff --git a/lark/visitors.py b/lark/visitors.py
index 0e051ed..b6db3ea 100644
--- a/lark/visitors.py
+++ b/lark/visitors.py
@@ -523,7 +523,7 @@ def v_args(inline: bool = False, meta: bool = False, tree: bool = False, wrapper
 
     Parameters:
         inline (bool, optional): Children are provided as ``*args`` instead of a list argument (not recommended for very long lists).
-        meta (bool, optional): Provides two arguments: ``meta`` and ``children`` (instead of just the latter)
+        meta (bool, optional): Provides two arguments: ``meta`` and ``children`` (instead of just the latter); ``meta`` isn't available for transformers supplied to Lark using the ``transformer`` parameter (aka internal transformers).
         tree (bool, optional): Provides the entire tree as the argument, instead of the children.
         wrapper (function, optional): Provide a function to decorate all methods.
 


Test patch:
diff --git a/tests/test_trees.py b/tests/test_trees.py
index 1f69869..82eceab 100644
--- a/tests/test_trees.py
+++ b/tests/test_trees.py
@@ -17,6 +17,11 @@ from lark.visitors import Visitor, Visitor_Recursive, Transformer, Interpreter,
 class TestTrees(TestCase):
     def setUp(self):
         self.tree1 = Tree('a', [Tree(x, y) for x, y in zip('bcd', 'xyz')])
+        self.tree2 = Tree('a', [
+            Tree('b', [Token('T', 'x')]),
+            Tree('c', [Token('T', 'y')]),
+            Tree('d', [Tree('z', [Token('T', 'zz'), Tree('zzz', 'zzz')])]),
+        ])
 
     def test_eq(self):
         assert self.tree1 == self.tree1
@@ -48,6 +53,11 @@ class TestTrees(TestCase):
         nodes = list(self.tree1.iter_subtrees_topdown())
         self.assertEqual(nodes, expected)
 
+    def test_find_token(self):
+        expected = [Token('T', 'x'), Token('T', 'y'), Token('T', 'zz')]
+        tokens = list(self.tree2.find_token('T'))
+        self.assertEqual(tokens, expected)
+
     def test_visitor(self):
         class Visitor1(Visitor):
             def __init__(self):
@@ -254,7 +264,7 @@ class TestTrees(TestCase):
             ab_partialmethod = partialmethod(ab_for_partialmethod, 1)
             set_union = set(["a"]).union
             static_add = staticmethod(add)
-            partial_reduce_mul = partial(reduce, mul)
+            partial_reduce_mul = staticmethod(partial(reduce, mul))
 
             custom_callable = CustomCallable()
 
@@ -355,8 +365,8 @@ class TestTrees(TestCase):
 
         @v_args(inline=True)
         class T(Transformer):
-            a = functools.partial(test, "@", postfix="!")
-            b = functools.partial(lambda s: s + "!")
+            a = staticmethod(functools.partial(test, "@", postfix="!"))
+            b = staticmethod(functools.partial(lambda s: s + "!"))
 
         res = T().transform(tree)
         assert res.children == ["@TEST1!", "test2!"]