Repository: python/mypy

Issue:
New semantic analyzer: remove redundant pass 3 / patching code
Much of the code of pass 3 is not required, but there may be checks that should be preserved. Some of the checks can probably be moved to the main pass.


Discussion:

Resolving patch:
diff --git a/mypy/newsemanal/semanal_pass3.py b/mypy/newsemanal/semanal_pass3.py
deleted file mode 100644
--- a/mypy/newsemanal/semanal_pass3.py
+++ /dev/null
@@ -1,194 +0,0 @@
-"""The semantic analyzer pass 3.
-
-This pass checks that type argument counts are valid; for example, it
-will reject Dict[int].  We don't do this in the second pass, since we
-infer the type argument counts of classes during this pass, and it is
-possible to refer to classes defined later in a file, which would not
-have the type argument count set yet. This pass also recomputes the
-method resolution order of each class, in case one of its bases
-belongs to a module involved in an import loop.
-"""
-
-from collections import OrderedDict
-from typing import Dict, List, Callable, Optional, Union, cast, Tuple
-
-from mypy import message_registry, state
-from mypy.nodes import (
-    Node, Expression, MypyFile, FuncDef, Decorator, RefExpr, Context, TypeInfo, ClassDef,
-    Block, TypedDictExpr, NamedTupleExpr, AssignmentStmt, IndexExpr, TypeAliasExpr, NameExpr,
-    CallExpr, NewTypeExpr, ForStmt, WithStmt, CastExpr, TypeVarExpr, TypeApplication, Lvalue,
-    TupleExpr, RevealExpr, SymbolTableNode, SymbolTable, Var, ARG_POS, OverloadedFuncDef,
-    MDEF, TypeAlias
-)
-from mypy.types import (
-    Type, Instance, AnyType, TypeOfAny, CallableType, TupleType, TypeVarType, TypedDictType,
-    UnionType, TypeType, Overloaded, ForwardRef, TypeTranslator, function_type, LiteralType,
-)
-from mypy.errors import Errors, report_internal_error
-from mypy.options import Options
-from mypy.traverser import TraverserVisitor
-from mypy.newsemanal.typeanal import TypeAnalyserPass3, collect_any_types
-from mypy.newsemanal.semanal import NewSemanticAnalyzer
-from mypy.scope import Scope
-from mypy.newsemanal.semanal_shared import SemanticAnalyzerCoreInterface
-
-
-class SemanticAnalyzerPass3(TraverserVisitor, SemanticAnalyzerCoreInterface):
-    """The third and final pass of semantic analysis.
-
-    Check type argument counts and values of generic types, and perform some
-    straightforward type inference.
-    """
-
-    def __init__(self, modules: Dict[str, MypyFile], errors: Errors,
-                 sem: NewSemanticAnalyzer) -> None:
-        self.modules = modules
-        self.errors = errors
-        self.sem = sem
-        self.scope = Scope()
-        # If True, process function definitions. If False, don't. This is used
-        # for processing module top levels in fine-grained incremental mode.
-        self.recurse_into_functions = True
-
-    def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
-                   patches: List[Tuple[int, Callable[[], None]]]) -> None:
-        self.recurse_into_functions = True
-        self.options = options
-        self.sem.options = options
-        self.patches = patches
-        self.is_typeshed_file = self.errors.is_typeshed_file(fnam)
-        self.sem.cur_mod_id = file_node.fullname()
-        self.cur_mod_node = file_node
-        self.sem.globals = file_node.names
-
-    def visit_class_def(self, tdef: ClassDef) -> None:
-        # NamedTuple base classes are validated in check_namedtuple_classdef; we don't have to
-        # check them again here.
-        self.scope.enter_class(tdef.info)
-        if not tdef.info.is_named_tuple:
-            types = list(tdef.info.bases)  # type: List[Type]
-            for tvar in tdef.type_vars:
-                if tvar.upper_bound:
-                    types.append(tvar.upper_bound)
-                if tvar.values:
-                    types.extend(tvar.values)
-            if tdef.info.tuple_type:
-                types.append(tdef.info.tuple_type)
-            self.analyze_types(types, tdef.info)
-        # Recompute MRO now that we have analyzed all modules, to pick
-        # up superclasses of bases imported from other modules in an
-        # import loop. (Only do so if we succeeded the first time.)
-        if tdef.info.mro:
-            tdef.info.mro = []  # Force recomputation
-            self.sem.calculate_class_mro(tdef)
-        super().visit_class_def(tdef)
-        self.analyze_symbol_table(tdef.info.names)
-        self.scope.leave()
-
-    def visit_decorator(self, dec: Decorator) -> None:
-        """Try to infer the type of the decorated function.
-
-        This lets us resolve references to decorated functions during
-        type checking when there are cyclic imports, as otherwise the
-        type might not be available when we need it.
-
-        This basically uses a simple special-purpose type inference
-        engine just for decorators.
-        """
-        # Don't just call the super method since we don't unconditionally traverse the decorated
-        # function.
-        dec.var.accept(self)
-        for decorator in dec.decorators:
-            decorator.accept(self)
-        if self.recurse_into_functions:
-            dec.func.accept(self)
-        self.analyze(dec.var.type, dec.var)
-
-    def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
-        """Traverse the assignment statement.
-
-        This includes the actual assignment and synthetic types
-        resulted from this assignment (if any). Currently this includes
-        NewType, TypedDict, NamedTuple, and TypeVar.
-        """
-        self.analyze(s.type, s)
-        if isinstance(s.rvalue, IndexExpr) and isinstance(s.rvalue.analyzed, TypeAliasExpr):
-            self.analyze(s.rvalue.analyzed.type, s.rvalue.analyzed, warn=True)
-        if isinstance(s.rvalue, CallExpr):
-            analyzed = s.rvalue.analyzed
-            if isinstance(analyzed, TypeVarExpr):
-                types = []
-                if analyzed.upper_bound:
-                    types.append(analyzed.upper_bound)
-                if analyzed.values:
-                    types.extend(analyzed.values)
-                self.analyze_types(types, analyzed)
-        if isinstance(s.lvalues[0], RefExpr) and isinstance(s.lvalues[0].node, Var):
-            self.analyze(s.lvalues[0].node.type, s.lvalues[0].node)
-        super().visit_assignment_stmt(s)
-
-    # Helpers
-
-    def analyze(self, type: Optional[Type], node: Node,
-                warn: bool = False) -> None:
-        # Recursive type warnings are only emitted on type definition 'node's, marked by 'warn'
-        # Flags appeared during analysis of 'type' are collected in this dict.
-        indicator = {}  # type: Dict[str, bool]
-        if type:
-            analyzer = self.make_type_analyzer(indicator)
-            type.accept(analyzer)
-            if analyzer.aliases_used:
-                target = self.scope.current_target()
-                self.cur_mod_node.alias_deps[target].update(analyzer.aliases_used)
-
-    def analyze_types(self, types: List[Type], node: Node) -> None:
-        # Similar to above but for nodes with multiple types.
-        indicator = {}  # type: Dict[str, bool]
-        for type in types:
-            analyzer = self.make_type_analyzer(indicator)
-            type.accept(analyzer)
-            if analyzer.aliases_used:
-                target = self.scope.current_target()
-                self.cur_mod_node.alias_deps[target].update(analyzer.aliases_used)
-
-    def analyze_symbol_table(self, names: SymbolTable) -> None:
-        """Analyze types in symbol table nodes only (shallow)."""
-        for node in names.values():
-            if isinstance(node.node, TypeAlias):
-                self.analyze(node.node.target, node.node)
-
-    def make_type_analyzer(self, indicator: Dict[str, bool]) -> TypeAnalyserPass3:
-        return TypeAnalyserPass3(self,
-                                 self.sem.plugin,
-                                 self.options,
-                                 self.is_typeshed_file,
-                                 indicator,
-                                 self.patches)
-
-    def lookup_qualified(self, name: str, ctx: Context,
-                         suppress_errors: bool = False) -> Optional[SymbolTableNode]:
-        return self.sem.lookup_qualified(name, ctx, suppress_errors=suppress_errors)
-
-    def lookup_fully_qualified(self, fullname: str) -> SymbolTableNode:
-        return self.sem.lookup_fully_qualified(fullname)
-
-    def fail(self, msg: str, ctx: Context, serious: bool = False, *,
-             blocker: bool = False) -> None:
-        self.sem.fail(msg, ctx, serious, blocker=blocker)
-
-    def fail_blocker(self, msg: str, ctx: Context) -> None:
-        self.fail(msg, ctx, blocker=True)
-
-    def note(self, msg: str, ctx: Context) -> None:
-        self.sem.note(msg, ctx)
-
-    def builtin_type(self, name: str, args: Optional[List[Type]] = None) -> Instance:
-        names = self.modules['builtins']
-        sym = names.names[name]
-        node = sym.node
-        assert isinstance(node, TypeInfo)
-        if args:
-            # TODO: assert len(args) == len(node.defn.type_vars)
-            return Instance(node, args)
-        any_type = AnyType(TypeOfAny.special_form)
-        return Instance(node, [any_type] * len(node.defn.type_vars))
diff --git a/mypy/newsemanal/typeanal.py b/mypy/newsemanal/typeanal.py
--- a/mypy/newsemanal/typeanal.py
+++ b/mypy/newsemanal/typeanal.py
@@ -847,162 +847,6 @@ def tuple_type(self, items: List[Type]) -> TupleType:
         return TupleType(items, fallback=self.named_type('builtins.tuple', [any_type]))
 
 
-class TypeAnalyserPass3(TypeVisitor[None]):
-    """Analyze type argument counts and values of generic types.
-
-    This is semantic analysis pass 3 for types.
-
-    Perform these operations:
-
-     * Report error for invalid type argument counts, such as List[x, y].
-     * Make implicit Any type arguments explicit my modifying types
-       in-place. For example, modify Foo into Foo[Any] if Foo expects a single
-       type argument.
-     * If a type variable has a value restriction, ensure that the value is
-       valid. For example, reject IO[int] if the type argument must be str
-       or bytes.
-
-    We can't do this earlier than the third pass, since type argument counts
-    are only determined in pass 2, and we have to support forward references
-    to types.
-    """
-
-    def __init__(self,
-                 api: SemanticAnalyzerCoreInterface,
-                 plugin: Plugin,
-                 options: Options,
-                 is_typeshed_stub: bool,
-                 indicator: Dict[str, bool],
-                 patches: List[Tuple[int, Callable[[], None]]]) -> None:
-        self.api = api
-        self.lookup_func = api.lookup_qualified
-        self.lookup_fqn_func = api.lookup_fully_qualified
-        self.fail = api.fail
-        self.note_func = api.note
-        self.options = options
-        self.plugin = plugin
-        self.is_typeshed_stub = is_typeshed_stub
-        self.indicator = indicator
-        self.patches = patches
-        self.aliases_used = set()  # type: Set[str]
-
-    def visit_instance(self, t: Instance) -> None:
-        info = t.type
-        if info.replaced or info.tuple_type:
-            self.indicator['synthetic'] = True
-        # Check type argument count.
-        if len(t.args) != len(info.type_vars):
-            fix_instance(t, self.fail)
-        elif info.defn.type_vars:
-            # Check type argument values. This is postponed to the end of semantic analysis
-            # since we need full MROs and resolved forward references.
-            for tvar in info.defn.type_vars:
-                if (tvar.values
-                        or not isinstance(tvar.upper_bound, Instance)
-                        or tvar.upper_bound.type.fullname() != 'builtins.object'):
-                    # Some restrictions on type variable. These can only be checked later
-                    # after we have final MROs and forward references have been resolved.
-                    self.indicator['typevar'] = True
-        for arg in t.args:
-            arg.accept(self)
-        if info.is_newtype:
-            for base in info.bases:
-                base.accept(self)
-
-    def visit_callable_type(self, t: CallableType) -> None:
-        t.ret_type.accept(self)
-        for arg_type in t.arg_types:
-            arg_type.accept(self)
-
-    def visit_overloaded(self, t: Overloaded) -> None:
-        for item in t.items():
-            item.accept(self)
-
-    def visit_tuple_type(self, t: TupleType) -> None:
-        for item in t.items:
-            item.accept(self)
-
-    def visit_typeddict_type(self, t: TypedDictType) -> None:
-        for item_type in t.items.values():
-            item_type.accept(self)
-
-    def visit_literal_type(self, t: LiteralType) -> None:
-        # We've already validated that the LiteralType
-        # contains either some literal expr like int, str, or
-        # bool in the previous pass -- we were able to do this
-        # since we had direct access to the underlying expression
-        # at those stages.
-        #
-        # The only thing we have left to check is to confirm
-        # whether LiteralTypes of the form 'Literal[Foo.bar]'
-        # contain enum members or not.
-        #
-        # TODO: implement this.
-        pass
-
-    def visit_union_type(self, t: UnionType) -> None:
-        for item in t.items:
-            item.accept(self)
-
-    def visit_star_type(self, t: StarType) -> None:
-        t.type.accept(self)
-
-    # Other kinds of type are trivial, since they are atomic (or invalid).
-
-    def visit_unbound_type(self, t: UnboundType) -> None:
-        pass
-
-    def visit_any(self, t: AnyType) -> None:
-        pass
-
-    def visit_none_type(self, t: NoneType) -> None:
-        pass
-
-    def visit_uninhabited_type(self, t: UninhabitedType) -> None:
-        pass
-
-    def visit_deleted_type(self, t: DeletedType) -> None:
-        pass
-
-    def visit_type_list(self, t: TypeList) -> None:
-        self.fail('Invalid type', t)
-
-    def visit_type_var(self, t: TypeVarType) -> None:
-        if t.upper_bound:
-            t.upper_bound.accept(self)
-        if t.values:
-            for v in t.values:
-                v.accept(self)
-
-    def visit_partial_type(self, t: PartialType) -> None:
-        pass
-
-    def visit_type_type(self, t: TypeType) -> None:
-        t.item.accept(self)
-
-    def visit_forwardref_type(self, t: ForwardRef) -> None:
-        self.indicator['forward'] = True
-        # mypyc plays badly with the janky failure to realize
-        # t.resolved is changed, so keep it from figuring out that it
-        # is None
-        if (t.resolved is None) is True:
-            resolved = self.anal_type(t.unbound)
-            t.resolve(resolved)
-            assert t.resolved is not None
-            t.resolved.accept(self)
-
-    def anal_type(self, tp: UnboundType) -> Type:
-        tpan = TypeAnalyser(self.api,
-                            None,
-                            self.plugin,
-                            self.options,
-                            self.is_typeshed_stub,
-                            third_pass=True)
-        res = tp.accept(tpan)
-        self.aliases_used = tpan.aliases_used
-        return res
-
-
 TypeVarList = List[Tuple[str, TypeVarExpr]]
 
 


Test patch: