Repository: asottile/pyupgrade

Issue:
IndentationError: unindent does not match any outer indentation level
To reproduce:
```python
# t.py

from __future__ import annotations

from typing import (
    Sequence,
    Union,
)


def sort_index(
    self,
    ascending: Union[Union[bool, int], Sequence[Union[bool, int]]],
):
    indexer = get_indexer_indexer(
        target, level, ascending, kind, na_position, sort_remaining, key
    )
```

Then:
```console
$ pyupgrade t.py 
Traceback (most recent call last):
  File "/home/marco/tmp/venv/bin/pyupgrade", line 8, in <module>
    sys.exit(main())
  File "/home/marco/tmp/venv/lib/python3.8/site-packages/pyupgrade/_main.py", line 889, in main
    ret |= _fix_file(filename, args)
  File "/home/marco/tmp/venv/lib/python3.8/site-packages/pyupgrade/_main.py", line 837, in _fix_file
    contents_text = _fix_tokens(contents_text, min_version=args.min_version)
  File "/home/marco/tmp/venv/lib/python3.8/site-packages/pyupgrade/_main.py", line 481, in _fix_tokens
    tokens = src_to_tokens(contents_text)
  File "/home/marco/tmp/venv/lib/python3.8/site-packages/tokenize_rt.py", line 68, in src_to_tokens
    for (
  File "/usr/lib/python3.8/tokenize.py", line 512, in _tokenize
    raise IndentationError(
  File "<tokenize>", line 15
    )
    ^
IndentationError: unindent does not match any outer indentation level
```

Removing either the `indexer = ` line, or the `ascending: ` annotation, makes `pyupgrade` pass. Keeping them both in, it fails.

Spotted in pandas, when commit 78371ccbd8744326e8535ac8bb9a661b073dfd88 meant that `pyupgrade` could no longer be run on `pandas/core/generic.py` without `--keep-runtime-typing`

----

version info:
```console
$ pip freeze | grep pyupgrade
pyupgrade==2.10.0
$ pip freeze | grep tokenize-rt
tokenize-rt==4.1.0
$ python --version
Python 3.8.5
```

Discussion:

Patch:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f92aa79..13c1a41 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -11,20 +11,20 @@ repos:
     -   id: requirements-txt-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/asottile/setup-cfg-fmt
-    rev: v1.17.0
+    rev: v1.16.0
     hooks:
     -   id: setup-cfg-fmt
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.0
+    rev: 3.8.4
     hooks:
     -   id: flake8
         additional_dependencies: [flake8-typing-imports==1.7.0]
 -   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.5.5
+    rev: v1.5.4
     hooks:
     -   id: autopep8
 -   repo: https://github.com/asottile/reorder_python_imports
-    rev: v2.4.0
+    rev: v2.3.6
     hooks:
     -   id: reorder-python-imports
         args: [--py3-plus]
@@ -34,11 +34,11 @@ repos:
     -   id: add-trailing-comma
         args: [--py36-plus]
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.10.0
+    rev: v2.8.0
     hooks:
     -   id: pyupgrade
         args: [--py36-plus]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.812
+    rev: v0.800
     hooks:
     -   id: mypy
diff --git a/README.md b/README.md
index de78502..2848fb3 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ Sample `.pre-commit-config.yaml`:
 
 ```yaml
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.10.0
+    rev: v2.8.0
     hooks:
     -   id: pyupgrade
 ```
@@ -358,10 +358,13 @@ six.ensure_text('...')                  # '...'
 Availability:
 - `--py3-plus` is passed on the commandline.
 
-```diff
--with io.open('f.txt') as f:
-+with open('f.txt') as f:
-     ...
+```python
+# input
+with io.open('f.txt') as f:
+    pass
+# output
+with open('f.txt') as f:
+    pass
 ```
 
 
@@ -476,27 +479,10 @@ Availability:
      ...
 ```
 
-
-### replace `@functools.lru_cache(maxsize=None)` with shorthand
-
-Availability:
-- `--py39-plus` is passed on the commandline.
-
-```diff
- import functools
-
--@functools.lru_cache(maxsize=None)
-+@functools.cache
- def expensive():
-     ...
-```
-
-
 ### pep 585 typing rewrites
 
 Availability:
 - File imports `from __future__ import annotations`
-    - Unless `--keep-runtime-typing` is passed on the commandline.
 - `--py39-plus` is passed on the commandline.
 
 ```diff
@@ -509,8 +495,7 @@ Availability:
 
 Availability:
 - File imports `from __future__ import annotations`
-    - Unless `--keep-runtime-typing` is passed on the commandline.
-- `--py310-plus` is passed on the commandline.
+- `--py39-plus` is passed on the commandline.
 
 ```diff
 -def f() -> Optional[str]:
diff --git a/pyupgrade/_data.py b/pyupgrade/_data.py
index 78923a9..7275c15 100644
--- a/pyupgrade/_data.py
+++ b/pyupgrade/_data.py
@@ -29,7 +29,6 @@ class Settings(NamedTuple):
     min_version: Version = (2, 7)
     keep_percent_format: bool = False
     keep_mock: bool = False
-    keep_runtime_typing: bool = False
 
 
 class State(NamedTuple):
diff --git a/pyupgrade/_main.py b/pyupgrade/_main.py
index c5628b1..bf75a62 100644
--- a/pyupgrade/_main.py
+++ b/pyupgrade/_main.py
@@ -830,8 +830,6 @@ def _fix_file(filename: str, args: argparse.Namespace) -> int:
         settings=Settings(
             min_version=args.min_version,
             keep_percent_format=args.keep_percent_format,
-            keep_mock=args.keep_mock,
-            keep_runtime_typing=args.keep_runtime_typing,
         ),
     )
     contents_text = _fix_tokens(contents_text, min_version=args.min_version)
@@ -857,7 +855,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
     parser.add_argument('--exit-zero-even-if-changed', action='store_true')
     parser.add_argument('--keep-percent-format', action='store_true')
     parser.add_argument('--keep-mock', action='store_true')
-    parser.add_argument('--keep-runtime-typing', action='store_true')
     parser.add_argument(
         '--py3-plus', '--py3-only',
         action='store_const', dest='min_version', default=(2, 7), const=(3,),
diff --git a/pyupgrade/_plugins/lru_cache.py b/pyupgrade/_plugins/lru_cache.py
index 4d2869f..0ef3abd 100644
--- a/pyupgrade/_plugins/lru_cache.py
+++ b/pyupgrade/_plugins/lru_cache.py
@@ -1,5 +1,4 @@
 import ast
-import functools
 from typing import Iterable
 from typing import List
 from typing import Tuple
@@ -12,7 +11,6 @@ from pyupgrade._ast_helpers import is_name_attr
 from pyupgrade._data import register
 from pyupgrade._data import State
 from pyupgrade._data import TokenFunc
-from pyupgrade._token_helpers import find_and_replace_call
 from pyupgrade._token_helpers import find_open_paren
 from pyupgrade._token_helpers import find_token
 
@@ -41,19 +39,3 @@ def visit_Call(
             )
     ):
         yield ast_to_offset(node), _remove_call
-    elif (
-            state.settings.min_version >= (3, 9) and
-            isinstance(node.func, ast.Attribute) and
-            node.func.attr == 'lru_cache' and
-            isinstance(node.func.value, ast.Name) and
-            node.func.value.id == 'functools' and
-            not node.args and
-            len(node.keywords) == 1 and
-            node.keywords[0].arg == 'maxsize' and
-            isinstance(node.keywords[0].value, ast.NameConstant) and
-            node.keywords[0].value.value is None
-    ):
-        func = functools.partial(
-            find_and_replace_call, template='functools.cache',
-        )
-        yield ast_to_offset(node), func
diff --git a/pyupgrade/_plugins/typing_pep585.py b/pyupgrade/_plugins/typing_pep585.py
index eb4d6cd..7a74066 100644
--- a/pyupgrade/_plugins/typing_pep585.py
+++ b/pyupgrade/_plugins/typing_pep585.py
@@ -16,16 +16,6 @@ PEP585_BUILTINS = frozenset((
 ))
 
 
-def _should_rewrite(state: State) -> bool:
-    return (
-        state.settings.min_version >= (3, 9) or (
-            not state.settings.keep_runtime_typing and
-            state.in_annotation and
-            'annotations' in state.from_imports['__future__']
-        )
-    )
-
-
 @register(ast.Attribute)
 def visit_Attribute(
         state: State,
@@ -33,7 +23,12 @@ def visit_Attribute(
         parent: ast.AST,
 ) -> Iterable[Tuple[Offset, TokenFunc]]:
     if (
-            _should_rewrite(state) and
+            (
+                state.settings.min_version >= (3, 9) or (
+                    state.in_annotation and
+                    'annotations' in state.from_imports['__future__']
+                )
+            ) and
             isinstance(node.value, ast.Name) and
             node.value.id == 'typing' and
             node.attr in PEP585_BUILTINS
@@ -53,7 +48,12 @@ def visit_Name(
         parent: ast.AST,
 ) -> Iterable[Tuple[Offset, TokenFunc]]:
     if (
-            _should_rewrite(state) and
+            (
+                state.settings.min_version >= (3, 9) or (
+                    state.in_annotation and
+                    'annotations' in state.from_imports['__future__']
+                )
+            ) and
             node.id in state.from_imports['typing'] and
             node.id in PEP585_BUILTINS
     ):
diff --git a/pyupgrade/_plugins/typing_pep604.py b/pyupgrade/_plugins/typing_pep604.py
index b5e22e1..880a8cb 100644
--- a/pyupgrade/_plugins/typing_pep604.py
+++ b/pyupgrade/_plugins/typing_pep604.py
@@ -5,6 +5,7 @@ from typing import Iterable
 from typing import List
 from typing import Tuple
 
+from tokenize_rt import NON_CODING_TOKENS
 from tokenize_rt import Offset
 from tokenize_rt import Token
 
@@ -35,40 +36,70 @@ def _fix_union(
         i: int,
         tokens: List[Token],
         *,
-        arg: ast.expr,
         arg_count: int,
 ) -> None:
-    arg_offset = ast_to_offset(arg)
-    j = find_token(tokens, i, '[')
-    to_delete = []
-    commas: List[int] = []
-
-    arg_depth = -1
     depth = 1
+    parens_done = []
+    open_parens = []
+    commas = []
+    coding_depth = None
+
+    j = find_token(tokens, i, '[')
     k = j + 1
     while depth:
+        # it's possible our first coding token is a close paren
+        # so make sure this is separate from the if chain below
+        if (
+                tokens[k].name not in NON_CODING_TOKENS and
+                tokens[k].src != '(' and
+                coding_depth is None
+        ):
+            if tokens[k].src == ')':  # the coding token was an empty tuple
+                coding_depth = depth - 1
+            else:
+                coding_depth = depth
+
         if tokens[k].src in OPENING:
-            if arg_depth == -1:
-                to_delete.append(k)
+            if tokens[k].src == '(':
+                open_parens.append((depth, k))
+
             depth += 1
         elif tokens[k].src in CLOSING:
+            if tokens[k].src == ')':
+                paren_depth, open_paren = open_parens.pop()
+                parens_done.append((paren_depth, (open_paren, k)))
+
             depth -= 1
-            if 0 < depth < arg_depth:
-                to_delete.append(k)
-        elif tokens[k].offset == arg_offset:
-            arg_depth = depth
-        elif depth == arg_depth and tokens[k].src == ',':
-            if len(commas) >= arg_count - 1:
-                to_delete.append(k)
-            else:
-                commas.append(k)
+        elif tokens[k].src == ',':
+            commas.append((depth, k))
 
         k += 1
     k -= 1
 
+    assert coding_depth is not None
+    assert not open_parens, open_parens
+    comma_depth = min((depth for depth, _ in commas), default=sys.maxsize)
+    min_depth = min(comma_depth, coding_depth)
+
+    to_delete = [
+        paren
+        for depth, positions in parens_done
+        if depth < min_depth
+        for paren in positions
+    ]
+
+    if comma_depth <= coding_depth:
+        comma_positions = [k for depth, k in commas if depth == comma_depth]
+        if len(comma_positions) == arg_count:
+            to_delete.append(comma_positions.pop())
+    else:
+        comma_positions = []
+
+    to_delete.sort()
+
     if tokens[j].line == tokens[k].line:
         del tokens[k]
-        for comma in commas:
+        for comma in comma_positions:
             tokens[comma] = Token('CODE', ' |')
         for paren in reversed(to_delete):
             del tokens[paren]
@@ -77,7 +108,7 @@ def _fix_union(
         tokens[j] = tokens[j]._replace(src='(')
         tokens[k] = tokens[k]._replace(src=')')
 
-        for comma in commas:
+        for comma in comma_positions:
             tokens[comma] = Token('CODE', ' |')
         for paren in reversed(to_delete):
             del tokens[paren]
@@ -87,10 +118,8 @@ def _fix_union(
 def _supported_version(state: State) -> bool:
     return (
         state.in_annotation and (
-            state.settings.min_version >= (3, 10) or (
-                not state.settings.keep_runtime_typing and
-                'annotations' in state.from_imports['__future__']
-            )
+            state.settings.min_version >= (3, 10) or
+            'annotations' in state.from_imports['__future__']
         )
     )
 
@@ -116,13 +145,11 @@ def visit_Subscript(
 
         if isinstance(node_slice, ast.Tuple):
             if node_slice.elts:
-                arg = node_slice.elts[0]
                 arg_count = len(node_slice.elts)
             else:
                 return  # empty Union
         else:
-            arg = node_slice
             arg_count = 1
 
-        func = functools.partial(_fix_union, arg=arg, arg_count=arg_count)
+        func = functools.partial(_fix_union, arg_count=arg_count)
         yield ast_to_offset(node), func
diff --git a/setup.cfg b/setup.cfg
index 440e085..2153948 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = pyupgrade
-version = 2.10.0
+version = 2.8.0
 description = A tool to automatically upgrade syntax for newer versions.
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -26,15 +26,15 @@ install_requires =
     tokenize-rt>=3.2.0
 python_requires = >=3.6.1
 
+[options.entry_points]
+console_scripts =
+    pyupgrade = pyupgrade._main:main
+
 [options.packages.find]
 exclude =
     tests*
     testing*
 
-[options.entry_points]
-console_scripts =
-    pyupgrade = pyupgrade._main:main
-
 [bdist_wheel]
 universal = True
 


Test patch:
diff --git a/tests/_plugins/__init__.py b/tests/_plugins/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/_plugins/typing_pep604_test.py b/tests/_plugins/typing_pep604_test.py
new file mode 100644
index 0000000..ca786b7
--- /dev/null
+++ b/tests/_plugins/typing_pep604_test.py
@@ -0,0 +1,28 @@
+import pytest
+from tokenize_rt import src_to_tokens
+from tokenize_rt import tokens_to_src
+
+from pyupgrade._plugins.typing_pep604 import _fix_union
+
+
+@pytest.mark.parametrize(
+    ('s', 'arg_count', 'expected'),
+    (
+        ('Union[a, b]', 2, 'a | b'),
+        ('Union[(a, b)]', 2, 'a | b'),
+        ('Union[(a,)]', 1, 'a'),
+        ('Union[(((a, b)))]', 2, 'a | b'),
+        pytest.param('Union[((a), b)]', 2, '(a) | b', id='wat'),
+        ('Union[(((a,), b))]', 2, '(a,) | b'),
+        ('Union[((a,), (a, b))]', 2, '(a,) | (a, b)'),
+        ('Union[((a))]', 1, 'a'),
+        ('Union[a()]', 1, 'a()'),
+        ('Union[a(b, c)]', 1, 'a(b, c)'),
+        ('Union[(a())]', 1, 'a()'),
+        ('Union[(())]', 1, '()'),
+    ),
+)
+def test_fix_union_edge_cases(s, arg_count, expected):
+    tokens = src_to_tokens(s)
+    _fix_union(0, tokens, arg_count=arg_count)
+    assert tokens_to_src(tokens) == expected
diff --git a/tests/features/lru_cache_test.py b/tests/features/lru_cache_test.py
index 5a37640..8c52bd5 100644
--- a/tests/features/lru_cache_test.py
+++ b/tests/features/lru_cache_test.py
@@ -82,47 +82,3 @@ def test_fix_no_arg_decorators_noop(s, min_version):
 def test_fix_no_arg_decorators(s, expected):
     ret = _fix_plugins(s, settings=Settings(min_version=(3, 8)))
     assert ret == expected
-
-
-@pytest.mark.parametrize(
-    ('s', 'min_version'),
-    (
-        pytest.param(
-            'from functools import lru_cache\n'
-            '@lru_cache(maxsize=None)\n'
-            'def foo(): pass\n',
-            (3, 9),
-            id='from imported',
-        ),
-        pytest.param(
-            'from functools import lru_cache\n'
-            '@lru_cache(maxsize=1024)\n'
-            'def foo(): pass\n',
-            (3, 9),
-            id='unrelated parameter',
-        ),
-    ),
-)
-def test_fix_maxsize_none_decorators_noop(s, min_version):
-    assert _fix_plugins(s, settings=Settings(min_version=min_version)) == s
-
-
-@pytest.mark.parametrize(
-    ('s', 'expected'),
-    (
-        pytest.param(
-            'import functools\n\n'
-            '@functools.lru_cache(maxsize=None)\n'
-            'def foo():\n'
-            '    pass\n',
-            'import functools\n\n'
-            '@functools.cache\n'
-            'def foo():\n'
-            '    pass\n',
-            id='call with attr',
-        ),
-    ),
-)
-def test_fix_maxsize_none_decorators(s, expected):
-    ret = _fix_plugins(s, settings=Settings(min_version=(3, 9)))
-    assert ret == expected
diff --git a/tests/features/typing_pep585_test.py b/tests/features/typing_pep585_test.py
index 7ac95e4..2982ad9 100644
--- a/tests/features/typing_pep585_test.py
+++ b/tests/features/typing_pep585_test.py
@@ -37,31 +37,6 @@ def test_fix_generic_types_noop(s, version):
     assert _fix_plugins(s, settings=Settings(min_version=version)) == s
 
 
-def test_noop_keep_runtime_typing():
-    s = '''\
-from __future__ import annotations
-from typing import List
-def f(x: List[str]) -> None: ...
-'''
-    settings = Settings(min_version=(3,), keep_runtime_typing=True)
-    assert _fix_plugins(s, settings=settings) == s
-
-
-def test_keep_runtime_typing_ignored_in_py39():
-    s = '''\
-from __future__ import annotations
-from typing import List
-def f(x: List[str]) -> None: ...
-'''
-    expected = '''\
-from __future__ import annotations
-from typing import List
-def f(x: list[str]) -> None: ...
-'''
-    settings = Settings(min_version=(3, 9), keep_runtime_typing=True)
-    assert _fix_plugins(s, settings=settings) == expected
-
-
 @pytest.mark.parametrize(
     ('s', 'expected'),
     (
diff --git a/tests/features/typing_pep604_test.py b/tests/features/typing_pep604_test.py
index 64c8f5d..0a21fd5 100644
--- a/tests/features/typing_pep604_test.py
+++ b/tests/features/typing_pep604_test.py
@@ -47,31 +47,6 @@ def test_fix_pep604_types_noop(s, version):
     assert _fix_plugins(s, settings=Settings(min_version=version)) == s
 
 
-def test_noop_keep_runtime_typing():
-    s = '''\
-from __future__ import annotations
-from typing import Union
-def f(x: Union[int, str]) -> None: ...
-'''
-    settings = Settings(min_version=(3,), keep_runtime_typing=True)
-    assert _fix_plugins(s, settings=settings) == s
-
-
-def test_keep_runtime_typing_ignored_in_py310():
-    s = '''\
-from __future__ import annotations
-from typing import Union
-def f(x: Union[int, str]) -> None: ...
-'''
-    expected = '''\
-from __future__ import annotations
-from typing import Union
-def f(x: int | str) -> None: ...
-'''
-    settings = Settings(min_version=(3, 10), keep_runtime_typing=True)
-    assert _fix_plugins(s, settings=settings) == expected
-
-
 @pytest.mark.parametrize(
     ('s', 'expected'),
     (
@@ -165,6 +140,15 @@ def f(x: int | str) -> None: ...
 
             id='Optional rewrite multi-line',
         ),
+        pytest.param(
+            'from typing import Union, Sequence\n'
+            'def f(x: Union[Union[A, B], Sequence[Union[C, D]]]): pass\n',
+
+            'from typing import Union, Sequence\n'
+            'def f(x: A | B | Sequence[C | D]): pass\n',
+
+            id='nested unions',
+        ),
     ),
 )
 def test_fix_pep604_types(s, expected):
diff --git a/tests/main_test.py b/tests/main_test.py
index 9f7f16b..da74a0c 100644
--- a/tests/main_test.py
+++ b/tests/main_test.py
@@ -82,15 +82,6 @@ def test_keep_percent_format(tmpdir):
     assert f.read() == '"{}".format(1)'
 
 
-def test_keep_mock(tmpdir):
-    f = tmpdir.join('f.py')
-    f.write('from mock import patch\n')
-    assert main((f.strpath, '--py3-plus', '--keep-mock')) == 0
-    assert f.read() == 'from mock import patch\n'
-    assert main((f.strpath, '--py3-plus')) == 1
-    assert f.read() == 'from unittest.mock import patch\n'
-
-
 def test_py3_plus_argument_unicode_literals(tmpdir):
     f = tmpdir.join('f.py')
     f.write('u""')