Repository: asottile/pyupgrade

Issue:
quoted annotations + `from __future__ import annotations`
should remove the quotes

need to be careful of:
- `Literal['foo']`
- `Annotated` ([PEP 593](https://www.python.org/dev/peps/pep-0593/))

Discussion:
MarcoGorelli: Thanks @asottile for your review,

I can address most comments, but the one I'm stuck on is

> the rest of the code tries to avoid recursion here as it can easily bump into the limit on deeply nested constructs (this was one of the motivations for the plugin-based system)

I think I can't just do something like override `visit_Name` or `visit_Subscript` because then I won't know whether the node is part of an assignment (which, barring exceptions like `Literal`, can be dequoted) or a type alias (which can't be dequoted). Any suggestions on how to address that or should I close?
MarcoGorelli: Thanks @asottile for your review - I've given this another go, and now there's a non-recursive solution, though it requires adding an extra attribute to `State`
MarcoGorelli: Thanks @asottile for explaining

Regarding

> I just meant implement your helper function non-recursively (use a local list / deque as a stack / queue)

I've tried to address that in the last commit, is this what you meant?
MarcoGorelli: Thanks you @asottile for your excellent review, I've attempted to address all comments in the latest commit
MarcoGorelli: Hey @asottile 

I'm still working on this, I just have a question - what's the expected output if someone uses incorrect syntax? e.g.
```python
x: NamedTuple('X', quox=[('foo', int)])
```
?
Should it be ignored, or should some error be thrown?
asottile: garbage should be ignored

Patch:
diff --git a/pyupgrade/_plugins/typing_pep563.py b/pyupgrade/_plugins/typing_pep563.py
new file mode 100644
index 0000000..33c05ab
--- /dev/null
+++ b/pyupgrade/_plugins/typing_pep563.py
@@ -0,0 +1,168 @@
+import ast
+import functools
+import sys
+from typing import Iterable
+from typing import List
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+
+from tokenize_rt import Offset
+
+from pyupgrade._ast_helpers import ast_to_offset
+from pyupgrade._data import register
+from pyupgrade._data import State
+from pyupgrade._data import Token
+from pyupgrade._data import TokenFunc
+
+
+def _supported_version(state: State) -> bool:
+    return (
+        state.settings.min_version >= (3, 10) or
+        'annotations' in state.from_imports['__future__']
+    )
+
+
+def _dequote(i: int, tokens: List[Token], *, new: str) -> None:
+    tokens[i] = tokens[i]._replace(src=new)
+
+
+def _get_name(node: ast.expr) -> str:
+    if isinstance(node, ast.Name):
+        return node.id
+    elif isinstance(node, ast.Attribute):
+        return node.attr
+    else:
+        raise AssertionError(f'expected Name or Attribute: {ast.dump(node)}')
+
+
+def _get_keyword_value(
+        keywords: List[ast.keyword],
+        keyword: str,
+) -> Optional[ast.expr]:
+    for kw in keywords:
+        if kw.arg == keyword:
+            return kw.value
+    else:
+        return None
+
+
+def _process_call(node: ast.Call) -> Iterable[ast.AST]:
+    name = _get_name(node.func)
+    args = node.args
+    keywords = node.keywords
+    if name == 'TypedDict':
+        if keywords:
+            for keyword in keywords:
+                yield keyword.value
+        elif len(args) != 2:  # garbage
+            pass
+        elif isinstance(args[1], ast.Dict):
+            yield from args[1].values
+        else:
+            raise AssertionError(f'expected ast.Dict: {ast.dump(args[1])}')
+    elif name == 'NamedTuple':
+        if len(args) == 2:
+            fields: Optional[ast.expr] = args[1]
+        elif keywords:
+            fields = _get_keyword_value(keywords, 'fields')
+        else:  # garbage
+            fields = None
+
+        if isinstance(fields, ast.List):
+            for elt in fields.elts:
+                if isinstance(elt, ast.Tuple) and len(elt.elts) == 2:
+                    yield elt.elts[1]
+        elif fields is not None:
+            raise AssertionError(f'expected ast.List: {ast.dump(fields)}')
+    elif name in {
+        'Arg',
+        'DefaultArg',
+        'NamedArg',
+        'DefaultNamedArg',
+        'VarArg',
+        'KwArg',
+    }:
+        if args:
+            yield args[0]
+        else:
+            keyword_value = _get_keyword_value(keywords, 'type')
+            if keyword_value is not None:
+                yield keyword_value
+
+
+def _process_subscript(node: ast.Subscript) -> Iterable[ast.AST]:
+    name = _get_name(node.value)
+    if name == 'Annotated':
+        if sys.version_info >= (3, 9):  # pragma: no cover (py39+)
+            node_slice: ast.expr = node.slice
+        elif isinstance(node.slice, ast.Index):  # pragma: no cover (<py39)
+            node_slice = node.slice.value
+        else:  # pragma: no cover (<py39)
+            pass  # unexpected slice type
+
+        if isinstance(node_slice, ast.Tuple):
+            if node_slice.elts:
+                yield node_slice.elts[0]
+        else:
+            raise AssertionError(f'expected ast.Tuple: {ast.dump(node_slice)}')
+    elif name != 'Literal':
+        yield node.slice
+
+
+def _replace_string_literal(
+        annotation: ast.expr,
+) -> Iterable[Tuple[Offset, TokenFunc]]:
+    nodes: List[ast.AST] = [annotation]
+    while nodes:
+        node = nodes.pop()
+        if isinstance(node, ast.Call):
+            nodes.extend(_process_call(node))
+        elif isinstance(node, ast.Subscript):
+            nodes.extend(_process_subscript(node))
+        elif isinstance(node, ast.Str):
+            func = functools.partial(_dequote, new=node.s)
+            yield ast_to_offset(node), func
+        else:
+            for name in node._fields:
+                value = getattr(node, name)
+                if isinstance(value, ast.AST):
+                    nodes.append(value)
+                elif isinstance(value, list):
+                    nodes.extend(value)
+
+
+def _process_args(
+        args: Sequence[Optional[ast.arg]],
+) -> Iterable[Tuple[Offset, TokenFunc]]:
+    for arg in args:
+        if arg is not None and arg.annotation is not None:
+            yield from _replace_string_literal(arg.annotation)
+
+
+@register(ast.FunctionDef)
+def visit_FunctionDef(
+        state: State,
+        node: ast.FunctionDef,
+        parent: ast.AST,
+) -> Iterable[Tuple[Offset, TokenFunc]]:
+    if not _supported_version(state):
+        return
+
+    yield from _process_args([node.args.vararg, node.args.kwarg])
+    yield from _process_args(node.args.args)
+    yield from _process_args(node.args.kwonlyargs)
+    yield from _process_args(getattr(node.args, 'posonlyargs', []))
+    if node.returns is not None:
+        yield from _replace_string_literal(node.returns)
+
+
+@register(ast.AnnAssign)
+def visit_AnnAssign(
+        state: State,
+        node: ast.AnnAssign,
+        parent: ast.AST,
+) -> Iterable[Tuple[Offset, TokenFunc]]:
+    if not _supported_version(state):
+        return
+    yield from _replace_string_literal(node.annotation)


Test patch:
diff --git a/tests/features/typing_pep563_test.py b/tests/features/typing_pep563_test.py
new file mode 100644
index 0000000..1196d0c
--- /dev/null
+++ b/tests/features/typing_pep563_test.py
@@ -0,0 +1,365 @@
+import sys
+
+import pytest
+
+from pyupgrade._data import Settings
+from pyupgrade._main import _fix_plugins
+
+
+@pytest.mark.parametrize(
+    ('s', 'version'),
+    (
+        pytest.param(
+            'from typing import Literal\n'
+            'x: "str"\n',
+            (2, 7),
+            id='not python 3.10+',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: Literal["foo", "bar"]\n',
+            (3,),
+            id='Literal',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x = TypeVar("x", "str")\n',
+            (3,),
+            id='TypeVar',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x = cast(x, "str")\n',
+            (3,),
+            id='cast',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'X = List["MyClass"]\n',
+            (3,),
+            id='Alias',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'X: MyCallable("X")\n',
+            (3,),
+            id='Custom callable',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'def foo(x, *args, **kwargs): ...\n',
+            (3,),
+            id='Untyped',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'def foo(*, inplace): ...\n',
+            (3,),
+            id='Kwonly, untyped',
+        ),
+    ),
+)
+def test_fix_typing_pep563_noop(s, version):
+    assert _fix_plugins(s, settings=Settings(min_version=version)) == s
+
+
+@pytest.mark.parametrize(
+    ('s', 'expected'),
+    (
+        pytest.param(
+            'from __future__ import annotations\n'
+            'def foo(var: "MyClass") -> "MyClass":\n'
+            '   x: "MyClass"\n',
+
+            'from __future__ import annotations\n'
+            'def foo(var: MyClass) -> MyClass:\n'
+            '   x: MyClass\n',
+
+            id='Simple annotation',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'def foo(*, inplace: "bool"): ...\n',
+
+            'from __future__ import annotations\n'
+            'def foo(*, inplace: bool): ...\n',
+
+            id='Kwonly, typed',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'def foo(*args: "str", **kwargs: "int"): ...\n',
+
+            'from __future__ import annotations\n'
+            'def foo(*args: str, **kwargs: int): ...\n',
+
+            id='Vararg and kwarg typed',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: Tuple["MyClass"]\n',
+
+            'from __future__ import annotations\n'
+            'x: Tuple[MyClass]\n',
+
+            id='Tuple',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: Callable[["MyClass"], None]\n',
+
+            'from __future__ import annotations\n'
+            'x: Callable[[MyClass], None]\n',
+
+            id='List within Callable',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'class Foo(NamedTuple):\n'
+            '    x: "MyClass"\n',
+
+            'from __future__ import annotations\n'
+            'class Foo(NamedTuple):\n'
+            '    x: MyClass\n',
+
+            id='Inherit from NamedTuple',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'class D(TypedDict):\n'
+            '    E: TypedDict("E", foo="int", total=False)\n',
+
+            'from __future__ import annotations\n'
+            'class D(TypedDict):\n'
+            '    E: TypedDict("E", foo=int, total=False)\n',
+
+            id='TypedDict keyword syntax',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'class D(TypedDict):\n'
+            '    E: TypedDict("E", {"foo": "int"})\n',
+
+            'from __future__ import annotations\n'
+            'class D(TypedDict):\n'
+            '    E: TypedDict("E", {"foo": int})\n',
+
+            id='TypedDict dict syntax',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'class D(typing.TypedDict):\n'
+            '    E: typing.TypedDict("E", {"foo": "int"})\n',
+
+            'from __future__ import annotations\n'
+            'class D(typing.TypedDict):\n'
+            '    E: typing.TypedDict("E", {"foo": int})\n',
+
+            id='typing.TypedDict',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'class D(TypedDict):\n'
+            '    E: TypedDict("E")\n',
+
+            'from __future__ import annotations\n'
+            'class D(TypedDict):\n'
+            '    E: TypedDict("E")\n',
+
+            id='TypedDict no type (invalid syntax)',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: Annotated["str", "metadata"]\n',
+
+            'from __future__ import annotations\n'
+            'x: Annotated[str, "metadata"]\n',
+
+            id='Annotated',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: typing.Annotated["str", "metadata"]\n',
+
+            'from __future__ import annotations\n'
+            'x: typing.Annotated[str, "metadata"]\n',
+
+            id='typing.Annotated',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: Annotated[()]\n',
+
+            'from __future__ import annotations\n'
+            'x: Annotated[()]\n',
+
+            id='Empty Annotated (garbage)',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: Arg("str", "name")\n',
+
+            'from __future__ import annotations\n'
+            'x: Arg(str, "name")\n',
+
+            id='Arg',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: DefaultArg("str", "name")\n',
+
+            'from __future__ import annotations\n'
+            'x: DefaultArg(str, "name")\n',
+
+            id='DefaultArg',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: NamedArg("str", "name")\n',
+
+            'from __future__ import annotations\n'
+            'x: NamedArg(str, "name")\n',
+
+            id='NamedArg',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg("str", "name")\n',
+
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg(str, "name")\n',
+
+            id='DefaultNamedArg',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg("str", name="name")\n',
+
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg(str, name="name")\n',
+
+            id='DefaultNamedArg with one keyword argument',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg(name="name", type="str")\n',
+
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg(name="name", type=str)\n',
+
+            id='DefaultNamedArg with keyword arguments',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg(name="name", quox="str")\n',
+
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg(name="name", quox="str")\n',
+
+            id='DefaultNamedArg with invalid arguments',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg(name="name")\n',
+
+            'from __future__ import annotations\n'
+            'x: DefaultNamedArg(name="name")\n',
+
+            id='DefaultNamedArg with no type (invalid syntax)',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: VarArg("str")\n',
+
+            'from __future__ import annotations\n'
+            'x: VarArg(str)\n',
+
+            id='VarArg',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: List[List[List["MyClass"]]]\n',
+
+            'from __future__ import annotations\n'
+            'x: List[List[List[MyClass]]]\n',
+
+            id='Nested',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: NamedTuple("X", [("foo", "int"), ("bar", "str")])\n',
+
+            'from __future__ import annotations\n'
+            'x: NamedTuple("X", [("foo", int), ("bar", str)])\n',
+
+            id='NamedTuple with types, no kwarg',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: NamedTuple("X", fields=[("foo", "int"), ("bar", "str")])\n',
+
+            'from __future__ import annotations\n'
+            'x: NamedTuple("X", fields=[("foo", int), ("bar", str)])\n',
+
+            id='NamedTuple with types, one kwarg',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: NamedTuple(typename="X", fields=[("foo", "int")])\n',
+
+            'from __future__ import annotations\n'
+            'x: NamedTuple(typename="X", fields=[("foo", int)])\n',
+
+            id='NamedTuple with types, two kwargs',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: NamedTuple("X", [("foo",), ("bar",)])\n',
+
+            'from __future__ import annotations\n'
+            'x: NamedTuple("X", [("foo",), ("bar",)])\n',
+
+            id='NamedTuple with length-1 tuples (invalid syntax)',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: NamedTuple("X", ["foo", "bar"])\n',
+
+            'from __future__ import annotations\n'
+            'x: NamedTuple("X", ["foo", "bar"])\n',
+
+            id='NamedTuple with missing types (invalid syntax)',
+        ),
+        pytest.param(
+            'from __future__ import annotations\n'
+            'x: NamedTuple()\n',
+
+            'from __future__ import annotations\n'
+            'x: NamedTuple()\n',
+
+            id='NamedTuple with no args (invalid syntax)',
+        ),
+    ),
+)
+def test_fix_typing_pep563(s, expected):
+    ret = _fix_plugins(s, settings=Settings(min_version=(3, 7)))
+    assert ret == expected
+
+
+@pytest.mark.xfail(
+    sys.version_info < (3, 8),
+    reason='posonly args not available in Python3.7',
+)
+def test_fix_typing_pep563_posonlyargs():
+    s = (
+        'from __future__ import annotations\n'
+        'def foo(var0, /, var1: "MyClass") -> "MyClass":\n'
+        '   x: "MyClass"\n'
+    )
+    expected = (
+        'from __future__ import annotations\n'
+        'def foo(var0, /, var1: MyClass) -> MyClass:\n'
+        '   x: MyClass\n'
+    )
+    ret = _fix_plugins(s, settings=Settings(min_version=(3, 8)))
+    assert ret == expected