Repository: python/mypy

Issue:
New semantic analyzer: Check type argument count for type aliases
This is a follow-up for https://github.com/python/mypy/pull/6395

Unfixed instances in type aliases cause new semantic analyzer to crash, for example:
```python
class C(Generic[T]):
    ...
def func(x: List[C[T]]) -> T:
    ...
A = List[C]  # Missing type argument for C
x: A

reveal_type(func(x))  # Crash here with list index out of range
```
See also TODO here https://github.com/python/mypy/blob/master/mypy/newsemanal/typeanal.py#L350





Discussion:
It looks like it is possible to get the crash from unfixed instance even with builtin aliases:
```python
x: Optional[List] = None  # incomplete instance here
y: List[str]

x = ['a', 'b']
x.extend(y)
```

Resolving patch:
diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py
--- a/mypy/newsemanal/semanal.py
+++ b/mypy/newsemanal/semanal.py
@@ -70,7 +70,7 @@
     YieldExpr, ExecStmt, BackquoteExpr, ImportBase, AwaitExpr,
     IntExpr, FloatExpr, UnicodeExpr, TempNode, OverloadPart,
     PlaceholderNode, COVARIANT, CONTRAVARIANT, INVARIANT,
-    LITERAL_YES, nongen_builtins, get_member_expr_fullname, REVEAL_TYPE,
+    nongen_builtins, get_member_expr_fullname, REVEAL_TYPE,
     REVEAL_LOCALS, is_final_node
 )
 from mypy.tvar_scope import TypeVarScope
@@ -84,12 +84,11 @@
     CallableType, Overloaded, Instance, Type, AnyType, LiteralType, LiteralValue,
     TypeTranslator, TypeOfAny, TypeType, NoneTyp, PlaceholderType
 )
-from mypy.type_visitor import TypeQuery
 from mypy.nodes import implicit_module_attrs
 from mypy.newsemanal.typeanal import (
     TypeAnalyser, analyze_type_alias, no_subscript_builtin_alias,
     TypeVariableQuery, TypeVarList, remove_dups, has_any_from_unimported_type,
-    check_for_explicit_any
+    check_for_explicit_any, fix_instance_types
 )
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
 from mypy.options import Options
@@ -101,7 +100,7 @@
 from mypy.util import get_prefix, correct_relative_import, unmangle, split_module_names
 from mypy.scope import Scope
 from mypy.newsemanal.semanal_shared import SemanticAnalyzerInterface, set_callable_name
-from mypy.newsemanal.semanal_namedtuple import NamedTupleAnalyzer, NAMEDTUPLE_PROHIBITED_NAMES
+from mypy.newsemanal.semanal_namedtuple import NamedTupleAnalyzer
 from mypy.newsemanal.semanal_typeddict import TypedDictAnalyzer
 from mypy.newsemanal.semanal_enum import EnumCallAnalyzer
 from mypy.newsemanal.semanal_newtype import NewTypeAnalyzer
@@ -2208,6 +2207,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None:
         # so we need to replace it with non-explicit Anys
         res = make_any_non_explicit(res)
         no_args = isinstance(res, Instance) and not res.args
+        fix_instance_types(res, self.fail)
         if isinstance(s.rvalue, (IndexExpr, CallExpr)):  # CallExpr is for `void = type(None)`
             s.rvalue.analyzed = TypeAliasExpr(res, alias_tvars, no_args)
             s.rvalue.analyzed.line = s.line
@@ -2343,6 +2343,9 @@ def make_name_lvalue_var(self, lvalue: NameExpr, kind: int, inferred: bool) -> V
             v.is_initialized_in_class = True
         if kind != LDEF:
             v._fullname = self.qualified_name(lvalue.name)
+        else:
+            # fullanme should never stay None
+            v._fullname = lvalue.name
         v.is_ready = False  # Type not inferred yet
         lvalue.is_new_def = True
         lvalue.is_inferred_def = True
@@ -3792,6 +3795,8 @@ def add_builtin_aliases(self, tree: MypyFile) -> None:
                     # Found built-in class target. Create alias.
                     target = self.named_type_or_none(target_name, [])
                     assert target is not None
+                    # Transform List to List[Any], etc.
+                    fix_instance_types(target, self.fail)
                     alias_node = TypeAlias(target, alias,
                                            line=-1, column=-1,  # there is no context
                                            no_args=True, normalized=True)
diff --git a/mypy/newsemanal/typeanal.py b/mypy/newsemanal/typeanal.py
--- a/mypy/newsemanal/typeanal.py
+++ b/mypy/newsemanal/typeanal.py
@@ -22,9 +22,10 @@
 from mypy.nodes import (
     TypeInfo, Context, SymbolTableNode, Var, Expression,
     IndexExpr, RefExpr, nongen_builtins, check_arg_names, check_arg_kinds, ARG_POS, ARG_NAMED,
-    ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, FuncDef, CallExpr, NameExpr,
-    Decorator, TypeAlias, MypyFile, PlaceholderNode
+    ARG_OPT, ARG_NAMED_OPT, ARG_STAR, ARG_STAR2, TypeVarExpr, CallExpr, NameExpr,
+    TypeAlias, PlaceholderNode
 )
+from mypy.typetraverser import TypeTraverserVisitor
 from mypy.tvar_scope import TypeVarScope
 from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError
 from mypy.plugin import Plugin, TypeAnalyzerPluginInterface, AnalyzeTypeContext
@@ -248,7 +249,13 @@ def visit_unbound_type_nonoptional(self, t: UnboundType) -> Type:
                 all_vars = node.alias_tvars
                 target = node.target
                 an_args = self.anal_array(t.args)
-                return expand_type_alias(target, all_vars, an_args, self.fail, node.no_args, t)
+                res = expand_type_alias(target, all_vars, an_args, self.fail, node.no_args, t)
+                # The only case where expand_type_alias() can return an incorrect instance is
+                # when it is top-level instance, so no need to recurse.
+                if (isinstance(res, Instance) and len(res.args) != len(res.type.type_vars) and
+                        not self.defining_alias):
+                    fix_instance(res, self.fail)
+                return res
             elif isinstance(node, TypeInfo):
                 return self.analyze_type_with_type_info(node, t.args, t)
             else:
@@ -347,7 +354,6 @@ def analyze_type_with_type_info(self, info: TypeInfo, args: List[Type], ctx: Con
         # Instance with an invalid number of type arguments.
         instance = Instance(info, self.anal_array(args), ctx.line, ctx.column)
         # Check type argument count.
-        # TODO: remove this from here and replace with a proper separate pass.
         if len(instance.args) != len(info.type_vars) and not self.defining_alias:
             fix_instance(instance, self.fail)
         if not args and self.options.disallow_any_generics and not self.defining_alias:
@@ -1281,3 +1287,22 @@ def make_optional_type(t: Type) -> Type:
         return UnionType(items + [NoneTyp()], t.line, t.column)
     else:
         return UnionType([t, NoneTyp()], t.line, t.column)
+
+
+def fix_instance_types(t: Type, fail: Callable[[str, Context], None]) -> None:
+    """Recursively fix all instance types (type argument count) in a given type.
+
+    For example 'Union[Dict, List[str, int]]' will be transformed into
+    'Union[Dict[Any, Any], List[Any]]' in place.
+    """
+    t.accept(InstanceFixer(fail))
+
+
+class InstanceFixer(TypeTraverserVisitor):
+    def __init__(self, fail: Callable[[str, Context], None]) -> None:
+        self.fail = fail
+
+    def visit_instance(self, typ: Instance) -> None:
+        super().visit_instance(typ)
+        if len(typ.args) != len(typ.type.type_vars):
+            fix_instance(typ, self.fail)


Test patch:
diff --git a/mypy/test/hacks.py b/mypy/test/hacks.py
--- a/mypy/test/hacks.py
+++ b/mypy/test/hacks.py
@@ -22,7 +22,6 @@
     'check-fastparse.test',
     'check-flags.test',
     'check-functions.test',
-    'check-generics.test',
     'check-incomplete-fixture.test',
     'check-incremental.test',
     'check-inference-context.test',
diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test
--- a/test-data/unit/check-generics.test
+++ b/test-data/unit/check-generics.test
@@ -579,8 +579,9 @@ reveal_type(wrap(z)) # E: Revealed type is '__main__.Node[builtins.int, builtins
 [out]
 main:13: error: Argument 2 to "Node" has incompatible type "int"; expected "str"
 
+-- Error formatting is a bit different (and probably better) with new analyzer
 [case testGenericTypeAliasesWrongAliases]
-# flags: --show-column-numbers --python-version 3.6 --no-strict-optional
+# flags: --show-column-numbers --python-version 3.6 --no-strict-optional --new-semantic-analyzer
 from typing import TypeVar, Generic, List, Callable, Tuple, Union
 T = TypeVar('T')
 S = TypeVar('S')
@@ -597,8 +598,8 @@ E = Node[Node[T, T], List[T]]
 F = Node[List[T, T], S] # Error
 G = Callable[..., List[T, T]] # Error
 H = Union[int, Tuple[T, Node[T]]] # Error
-h: H # Error
-h1: H[int, str] # Two errors here, wrong number of args for H, and for Node
+h: H # This was reported on previous line
+h1: H[int, str] # Error
 
 x = None # type: D[int, str]
 reveal_type(x)
@@ -614,9 +615,7 @@ main:11:5: error: "Node" expects 2 type arguments, but 3 given
 main:15:10: error: "list" expects 1 type argument, but 2 given
 main:16:19: error: "list" expects 1 type argument, but 2 given
 main:17:25: error: "Node" expects 2 type arguments, but 1 given
-main:18:4: error: "Node" expects 2 type arguments, but 1 given
 main:19:5: error: Bad number of arguments for type alias, expected: 1, given: 2
-main:19:5: error: "Node" expects 2 type arguments, but 1 given
 main:22:1: error: Revealed type is '__main__.Node[builtins.int, builtins.str]'
 main:24:1: error: Revealed type is '__main__.Node[__main__.Node[builtins.int, builtins.int], builtins.list[builtins.int]]'
 main:26:5: error: Type variable "__main__.T" is invalid as target for type alias
@@ -1480,7 +1479,10 @@ class A(Generic[T]):
         g(self.a)
         g(n) # E: Argument 1 to "g" has incompatible type "int"; expected "T"
 
+-- This is non-trivial with new analyzer (and also in fine grained incremental):
+-- We need to store whole tvar_scope, not only active class.
 [case testFunctionInGenericInnerClassTypeVariable]
+# flags: --no-new-semantic-analyzer
 from typing import TypeVar, Generic
 
 T = TypeVar('T')
diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test
--- a/test-data/unit/check-newsemanal.test
+++ b/test-data/unit/check-newsemanal.test
@@ -1400,6 +1400,50 @@ reveal_type(a.x)  # E: Revealed type is '__main__.B'
 a.y = 1  # E: Incompatible types in assignment (expression has type "int", variable has type "B")
 [builtins fixtures/property.pyi]
 
+[case testNewAnalyzerAliasesFixedFew]
+from typing import List, Generic, TypeVar
+
+def func(x: List[C[T]]) -> T:
+    ...
+x: A
+A = List[C]
+
+reveal_type(x)  # E: Revealed type is 'builtins.list[__main__.C[Any]]'
+reveal_type(func(x))  # E: Revealed type is 'Any'
+class C(Generic[T]):
+    ...
+
+T = TypeVar('T')
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasesFixedMany]
+from typing import List, Generic, TypeVar
+
+def func(x: List[C[T]]) -> T:
+    ...
+
+x: A
+A = List[C[int, str]]  # E: "C" expects 1 type argument, but 2 given
+
+reveal_type(x)  # E: Revealed type is 'builtins.list[__main__.C[Any]]'
+reveal_type(func(x))  # E: Revealed type is 'Any'
+
+class C(Generic[T]):
+    ...
+T = TypeVar('T')
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerBuiltinAliasesFixed]
+from typing import List, Optional
+x: Optional[List] = None
+y: List[str]
+
+reveal_type(x)  # E: Revealed type is 'Union[builtins.list[Any], None]'
+x = ['a', 'b']
+reveal_type(x)  # E: Revealed type is 'builtins.list[Any]'
+x.extend(y)
+[builtins fixtures/list.pyi]
+
 [case testNewAnalyzerImportPriosB]
 import b
 [file a.py]