Repository: wemake-services/wemake-python-styleguide

Issue:
`self._tokens[1]`: `IndexError: list index out of range`
https://github.com/dry-python/returns/actions/runs/8437642376/job/23107904196?pr=1810

```

Run poetry run flake8 .
Traceback (most recent call last):
  File "/home/runner/work/returns/returns/.venv/lib/python3.11/site-packages/wemake_python_styleguide/checker.py", line 154, in run
    visitor.run()
  File "/home/runner/work/returns/returns/.venv/lib/python3.11/site-packages/wemake_python_styleguide/visitors/base.py", line 286, in run
    self.visit(token)
  File "/home/runner/work/returns/returns/.venv/lib/python3.11/site-packages/wemake_python_styleguide/visitors/ast/function_empty_lines.py", line 150, in visit
    ).analyze()
      ^^^^^^^^^
  File "/home/runner/work/returns/returns/.venv/lib/python3.11/site-packages/wemake_python_styleguide/visitors/ast/function_empty_lines.py", line 114, in analyze
    function.name_token(),
    ^^^^^^^^^^^^^^^^^^^^^
  File "/home/runner/work/returns/returns/.venv/lib/python3.11/site-packages/wemake_python_styleguide/visitors/ast/function_empty_lines.py", line 18, in name_token
    return self._tokens[1]
           ~~~~~~~~~~~~^^^
IndexError: list index out of range
```




Discussion:

Resolving patch:
diff --git a/wemake_python_styleguide/presets/types/tree.py b/wemake_python_styleguide/presets/types/tree.py
--- a/wemake_python_styleguide/presets/types/tree.py
+++ b/wemake_python_styleguide/presets/types/tree.py
@@ -11,7 +11,6 @@
     conditions,
     decorators,
     exceptions,
-    function_empty_lines,
     functions,
     imports,
     iterables,
@@ -23,6 +22,9 @@
     statements,
     subscripts,
 )
+from wemake_python_styleguide.visitors.tokenize import (
+    functions as tokenize_functions,
+)
 
 #: Used to store all general visitors to be later passed to checker:
 PRESET: Final = (
@@ -57,6 +59,7 @@
     functions.UnnecessaryLiteralsVisitor,
     functions.FunctionSignatureVisitor,
     functions.FloatingNanCallVisitor,
+    tokenize_functions.WrongEmptyLinesCountVisitor,
 
     exceptions.WrongTryExceptVisitor,
     exceptions.NestedTryBlocksVisitor,
@@ -110,8 +113,6 @@
 
     redundancy.RedundantEnumerateVisitor,
 
-    function_empty_lines.WrongEmptyLinesCountVisitor,
-
     # Modules:
     modules.EmptyModuleContentsVisitor,
     modules.MagicModuleFunctionsVisitor,
diff --git a/wemake_python_styleguide/visitors/ast/function_empty_lines.py b/wemake_python_styleguide/visitors/tokenize/functions.py
similarity index 74%
rename from wemake_python_styleguide/visitors/ast/function_empty_lines.py
rename to wemake_python_styleguide/visitors/tokenize/functions.py
--- a/wemake_python_styleguide/visitors/ast/function_empty_lines.py
+++ b/wemake_python_styleguide/visitors/tokenize/functions.py
@@ -1,6 +1,6 @@
 import math
 import tokenize
-from typing import Iterator, List
+from typing import Iterable, List, Optional, Tuple
 
 from typing_extensions import final
 
@@ -36,23 +36,19 @@ def _is_target_line(self, token: tokenize.TokenInfo) -> bool:
 
 @final
 class _FileFunctions:
-
     def __init__(self, file_tokens: List[tokenize.TokenInfo]) -> None:
         self._file_tokens = file_tokens
 
-    def as_list(self) -> List[_Function]:
-        return list(self._search_functions())
-
-    def _search_functions(self) -> Iterator[_Function]:
+    def search_functions(self) -> Iterable[_Function]:  # noqa: WPS210
         function_tokens: List[tokenize.TokenInfo] = []
         in_function = False
         function_start_token = (0, 0)
-        for token in self._file_tokens:
+        for token_index, token in enumerate(self._file_tokens):
             function_ended = self._is_function_end(
                 token,
-                bool(function_tokens),
-                function_start_token[1],
-                function_start_token[0],
+                token_index,
+                function_start_token,
+                function_tokens_exists=bool(function_tokens),
             )
             if not in_function and self._is_function_start(token):
                 in_function = True
@@ -71,18 +67,33 @@ def _is_function_start(self, token: tokenize.TokenInfo) -> bool:
     def _is_function_end(
         self,
         token: tokenize.TokenInfo,
+        token_index: int,
+        function_start: Tuple[int, int],
+        *,
         function_tokens_exists: bool,
-        function_start_column: int,
-        function_start_line: int,
     ) -> bool:
-        is_elipsis = token.string == '...'
-        is_elipsis_end = is_elipsis and token.start[0] == function_start_line
+        next_token = self._next_token(token_index)
+        is_elipsis_end = (
+            next_token and
+            next_token.exact_type == tokenize.NEWLINE and
+            token.string == '...' and
+            token.start[0] == function_start[0]
+        )
         if is_elipsis_end:
             return True
-        column_valid = token.start[1] in {0, function_start_column}
+        column_valid = token.start[1] in {0, function_start[1]}
         is_dedent_token = token.type == tokenize.DEDENT
         return is_dedent_token and function_tokens_exists and column_valid
 
+    def _next_token(
+        self,
+        token_index: int,
+    ) -> Optional[tokenize.TokenInfo]:
+        try:
+            return self._file_tokens[token_index + 1]
+        except IndexError:
+            return None
+
 
 @final
 class _FileTokens:
@@ -95,12 +106,13 @@ def __init__(
         self._file_functions = file_functions
         self._exps_for_one_empty_line = exps_for_one_empty_line
 
-    def analyze(self) -> List[best_practices.WrongEmptyLinesCountViolation]:
-        violations = []
-        for function in self._file_functions.as_list():
+    def analyze(self) -> Iterable[best_practices.WrongEmptyLinesCountViolation]:
+        for function in self._file_functions.search_functions():
             splitted_function_body = function.body().strip().split('\n')
             empty_lines_count = len([
-                line for line in splitted_function_body if line == ''
+                line
+                for line in splitted_function_body
+                if line == ''
             ])
             if not empty_lines_count:
                 continue
@@ -109,14 +121,11 @@ def analyze(self) -> List[best_practices.WrongEmptyLinesCountViolation]:
                 len(splitted_function_body), empty_lines_count,
             )
             if empty_lines_count > available_empty_lines:
-                violations.append(
-                    best_practices.WrongEmptyLinesCountViolation(
-                        function.name_token(),
-                        text=str(empty_lines_count),
-                        baseline=available_empty_lines,
-                    ),
+                yield best_practices.WrongEmptyLinesCountViolation(
+                    function.name_token(),
+                    text=str(empty_lines_count),
+                    baseline=available_empty_lines,
                 )
-        return violations
 
     def _available_empty_lines(
         self,


Test patch:
diff --git a/tests/test_visitors/test_ast/test_classes/test_wrong_empty_lines_count.py b/tests/test_visitors/test_ast/test_classes/test_wrong_empty_lines_count.py
--- a/tests/test_visitors/test_ast/test_classes/test_wrong_empty_lines_count.py
+++ b/tests/test_visitors/test_ast/test_classes/test_wrong_empty_lines_count.py
@@ -4,7 +4,7 @@
 from wemake_python_styleguide.violations.best_practices import (
     WrongEmptyLinesCountViolation,
 )
-from wemake_python_styleguide.visitors.ast.function_empty_lines import (
+from wemake_python_styleguide.visitors.tokenize.functions import (
     WrongEmptyLinesCountVisitor,
 )
 
@@ -228,6 +228,16 @@ def kk() -> None:
     )
 """
 
+# https://github.com/wemake-services/wemake-python-styleguide/issues/2899
+regression2899_1 = """
+def curry(function: Callable[..., _ReturnType]) -> Callable[..., _ReturnType]:
+    ...
+"""
+
+regression2899_2 = """
+def cur(function: Callable[..., _ReturnType]) -> Callable[..., _ReturnType]: ...
+"""
+
 
 @pytest.mark.parametrize('input_', [
     class_with_wrong_method,
@@ -262,6 +272,10 @@ def test_wrong(
     class_with_attributes,
     expression_without_function,
     module_level_empty_lines,
+
+    # Do not report anything for `Callable[...]`
+    regression2899_1,
+    regression2899_2,
 ])
 def test_success(
     template,
@@ -304,7 +318,6 @@ def test_zero_option(
 def test_zero_option_with_valid_method(
     template,
     parse_tokens,
-    default_options,
     assert_errors,
     options,
     mode,
@@ -357,7 +370,6 @@ def test_string_concatination(
     parse_tokens,
     default_options,
     assert_errors,
-    options,
     mode,
 ):
     """Test function with multiline implicit string concatenation."""