Repository: google/pytype

Issue:
merge-pyi feature request: do not annotate builtin functions
After using merge-pyi, the code base is littered with new annotations to `__str__`, `__eq__`, `__hash__` etc., which I don't bother annotating because they have well known data types. Would it be possible to not annotate these methods, when they match the expected builtin types?
Skips built in functions while merging pyi.
Fixes #550.


Discussion:
I think we could skip methods that start and end with `__` by adding a check here: https://github.com/google/pytype/blob/51bcd3a2d7be9db0af39a64074cd5ea4917eee4c/pytype/tools/merge_pyi/merge_pyi.py#L628


Resolving patch:
diff --git a/pytype/abstract.py b/pytype/abstract.py
--- a/pytype/abstract.py
+++ b/pytype/abstract.py
@@ -1,3 +1,4 @@
+# Lint as: python3
 """The abstract values used by vm.py.
 
 This file contains AtomicAbstractValue and its subclasses. Mixins such as Class
@@ -1441,7 +1442,7 @@ class Function(SimpleAbstractValue):
 
   def __init__(self, name, vm):
     super(Function, self).__init__(name, vm)
-    self.cls = self.vm.convert.function_type
+    self.cls = FunctionPyTDClass(self, vm)
     self.is_attribute_of_class = False
     self.is_abstract = False
     self.members["func_name"] = self.vm.convert.build_string(
@@ -2409,6 +2410,24 @@ def generate_ast(self):
         template=self.pytd_cls.template)
 
 
+class FunctionPyTDClass(PyTDClass):
+  """PyTDClass(Callable) subclass to support annotating higher-order functions.
+
+  In InterpreterFunction calls, type parameter annotations are handled by
+  getting the types of the parameters from the arguments and instantiating them
+  in the return value. To handle a signature like (func: T) -> T, we need to
+  save the value of `func`, not just its type of Callable.
+  """
+
+  def __init__(self, func, vm):
+    super().__init__("typing.Callable", vm.convert.function_type.pytd_cls, vm)
+    self.func = func
+
+  def instantiate(self, node, container=None):
+    del container  # unused
+    return self.func.to_variable(node)
+
+
 class InterpreterClass(SimpleAbstractValue, mixin.Class):
   """An abstract wrapper for user-defined class objects.
 
@@ -2649,7 +2668,7 @@ def call(self, node, _, args, alias_map=None):
     namedargs = {k: u.AssignToNewVariable(node)
                  for k, u in args.namedargs.items()}
     try:
-      inspect.getcallargs(self.func, node, *posargs, **namedargs)
+      inspect.signature(self.func).bind(node, *posargs, **namedargs)
     except ValueError:
       # Happens for, e.g.,
       #   def f((x, y)): pass
@@ -2669,7 +2688,7 @@ def call(self, node, _, args, alias_map=None):
       # The function was passed the wrong number of arguments. The signature is
       # ([self, ]node, ...). The length of "..." tells us how many variables
       # are expected.
-      expected_argcount = len(inspect.getargspec(self.func).args) - 1
+      expected_argcount = len(inspect.getfullargspec(self.func).args) - 1
       if inspect.ismethod(self.func) and self.func.__self__ is not None:
         expected_argcount -= 1
       actual_argcount = len(posargs) + len(namedargs)
diff --git a/pytype/annotations_util.py b/pytype/annotations_util.py
--- a/pytype/annotations_util.py
+++ b/pytype/annotations_util.py
@@ -107,18 +107,25 @@ def add_scope(self, annot, types, module):
       return annot
     return annot
 
-  def get_type_parameters(self, annot):
+  def get_type_parameters(self, annot, seen=None):
     """Get all the TypeParameter instances that appear in the annotation."""
+    seen = seen or set()
+    if annot in seen:
+      return []
+    if isinstance(annot, abstract.ParameterizedClass):
+      # We track parameterized classes to avoid recursion errors when a class
+      # contains itself.
+      seen |= {annot}
     if isinstance(annot, abstract.TypeParameter):
       return [annot]
     elif isinstance(annot, abstract.TupleClass):
       return self.get_type_parameters(
-          annot.formal_type_parameters[abstract_utils.T])
+          annot.formal_type_parameters[abstract_utils.T], seen)
     elif isinstance(annot, abstract.ParameterizedClass):
-      return sum((self.get_type_parameters(p)
+      return sum((self.get_type_parameters(p, seen)
                   for p in annot.formal_type_parameters.values()), [])
     elif isinstance(annot, abstract.Union):
-      return sum((self.get_type_parameters(o) for o in annot.options), [])
+      return sum((self.get_type_parameters(o, seen) for o in annot.options), [])
     return []
 
   def convert_function_type_annotation(self, name, typ):
diff --git a/pytype/tools/merge_pyi/merge_pyi.py b/pytype/tools/merge_pyi/merge_pyi.py
--- a/pytype/tools/merge_pyi/merge_pyi.py
+++ b/pytype/tools/merge_pyi/merge_pyi.py
@@ -626,6 +626,13 @@ def insert_comment_annotation(self, src_sig, pyi_sig):
       return []
 
   def can_annotate(self, src_sig):
+    if (src_sig.short_name.startswith('__') and
+        src_sig.short_name.endswith('__') and
+        src_sig.short_name != '__init__' and
+        src_sig.short_name != '__new__'):
+      self.logger.info('magic method, skipping %s', src_sig)
+      return False
+
     if src_sig.has_pep484_annotations or src_sig.has_comment_annotations:
       self.logger.warning('already annotated, skipping %s', src_sig)
       return False


Test patch:
diff --git a/pytype/tests/CMakeLists.txt b/pytype/tests/CMakeLists.txt
--- a/pytype/tests/CMakeLists.txt
+++ b/pytype/tests/CMakeLists.txt
@@ -82,6 +82,7 @@ py_test(
     test_functions.py
   DEPS
     .test_base
+    pytype.pytd_utils
 )
 
 py_test(
diff --git a/pytype/tests/py3/test_annotations.py b/pytype/tests/py3/test_annotations.py
--- a/pytype/tests/py3/test_annotations.py
+++ b/pytype/tests/py3/test_annotations.py
@@ -3,6 +3,7 @@
 
 from pytype import file_utils
 from pytype.tests import test_base
+from pytype.tests import test_utils
 
 
 class AnnotationTest(test_base.TargetPython3BasicTest):
@@ -1022,11 +1023,29 @@ class TestAnnotationsPython3Feature(test_base.TargetPython3FeatureTest):
 
   def testVariableAnnotations(self):
     ty = self.Infer("""
-      a : int = 42
+      a: int = 42
     """, deep=False)
     self.assertTypesMatchPytd(ty, """
       a: int
     """)
 
+  @test_utils.skipUnlessPy((3, 7), reason="__future__.annotations is 3.7+ and "
+                           "is the default behavior in 3.8+")
+  def testPostponedEvaluation(self):
+    self.Check("""
+      from __future__ import annotations
+      def f() -> int:
+        return 0
+    """)
+
+  @test_utils.skipUnlessPy((3, 7), reason="__future__.annotations is 3.7+ and "
+                           "is the default behavior in 3.8+")
+  def testPostponedEvaluationError(self):
+    self.CheckWithErrors("""
+      from __future__ import annotations
+      def f() -> str:
+        return 0  # bad-return-type
+    """)
+
 
 test_base.main(globals(), __name__ == "__main__")
diff --git a/pytype/tests/py3/test_typing.py b/pytype/tests/py3/test_typing.py
--- a/pytype/tests/py3/test_typing.py
+++ b/pytype/tests/py3/test_typing.py
@@ -3,6 +3,7 @@
 from pytype import file_utils
 from pytype.pytd import pep484
 from pytype.tests import test_base
+from pytype.tests import test_utils
 
 
 class TypingTest(test_base.TargetPython3BasicTest):
@@ -647,5 +648,14 @@ class A(object):
         x: int = ...
     """)
 
+  @test_utils.skipBeforePy((3, 7), "typing.OrderedDict is new in 3.7")
+  def test_ordered_dict(self):
+    self.Check("""
+      import collections
+      from typing import OrderedDict
+      def f(x: OrderedDict[str, int]): ...
+      f(collections.OrderedDict(a=0))
+    """)
+
 
 test_base.main(globals(), __name__ == "__main__")
diff --git a/pytype/tests/test_base_test.py b/pytype/tests/test_base_test.py
--- a/pytype/tests/test_base_test.py
+++ b/pytype/tests/test_base_test.py
@@ -141,5 +141,23 @@ def test_skip_if_py(self):
         name_error
     """)
 
+  @test_utils.skipBeforePy((3, 7), reason="testing skipBeforePy")
+  def test_skip_before_py(self):
+    # This will fail before 3.7.
+    self.Check("""
+      import sys
+      if sys.version_info.minor < 7:
+        name_error
+    """)
+
+  @test_utils.skipFromPy((3, 7), reason="testing skipFromPy")
+  def test_skip_from_py(self):
+    # This will fail in 3.7+.
+    self.Check("""
+      import sys
+      if sys.version_info.minor >= 7:
+        name_error
+    """)
+
 
 test_base.main(globals(), __name__ == "__main__")
diff --git a/pytype/tests/test_functions.py b/pytype/tests/test_functions.py
--- a/pytype/tests/test_functions.py
+++ b/pytype/tests/test_functions.py
@@ -1,6 +1,7 @@
 """Test functions."""
 
 from pytype import file_utils
+from pytype.pytd import pytd_utils
 from pytype.tests import test_base
 
 
@@ -681,13 +682,13 @@ def f(): pass
         w2 = type(f)
       """, pythonpath=[d.path])
       self.assertTypesMatchPytd(ty, """
-        from typing import Any, Callable, Tuple, Type
+        from typing import Any, Callable, Tuple
         foo = ...  # type: module
         def f() -> None: ...
         v1 = ...  # type: Tuple[Callable[[], None]]
-        v2 = ...  # type: Type[Callable]
+        v2 = Callable
         w1 = ...  # type: Tuple[Callable[[], Any]]
-        w2 = ...  # type: Type[Callable]
+        w2 = Callable
       """)
 
   def test_type_parameter_visibility(self):
@@ -972,5 +973,34 @@ def test_bad_comprehensions(self):
       lambda x: name_error4  # name-error
     """)
 
+  def test_identity_decorator(self):
+    ty = self.Infer("""
+      from typing import TypeVar
+      T = TypeVar("T")
+      def decorate(func: T) -> T:
+        return func
+      @decorate
+      def f():
+        return 0
+    """)
+    self.assertTypesMatchPytd(ty, """
+      from typing import TypeVar
+      T = TypeVar("T")
+      def decorate(func: T) -> T: ...
+      def f() -> int: ...
+    """)
+    with file_utils.Tempdir() as d:
+      d.create_file("foo.pyi", pytd_utils.Print(ty))
+      ty2 = self.Infer("""
+        import foo
+        @foo.decorate
+        def g():
+          return ""
+      """, pythonpath=[d.path])
+    self.assertTypesMatchPytd(ty2, """
+      foo: module
+      def g() -> str: ...
+    """)
+
 
 test_base.main(globals(), __name__ == "__main__")
diff --git a/pytype/tests/test_typevar.py b/pytype/tests/test_typevar.py
--- a/pytype/tests/test_typevar.py
+++ b/pytype/tests/test_typevar.py
@@ -410,5 +410,13 @@ def test_typevar_in_alias(self):
     self.assertErrorRegexes(
         err, {"e": "aliases of Unions with type parameters"})
 
+  def test_recursive_alias(self):
+    self.Check("""
+      from typing import Any, Iterable, TypeVar, Union
+      T = TypeVar("T")
+      X = Union[Any, Iterable["X"]]
+      Y = Union[Any, X]
+    """)
+
 
 test_base.main(globals(), __name__ == "__main__")
diff --git a/pytype/tests/test_utils.py b/pytype/tests/test_utils.py
--- a/pytype/tests/test_utils.py
+++ b/pytype/tests/test_utils.py
@@ -534,4 +534,12 @@ def skipIfPy(*versions, reason):
 
 def skipUnlessPy(*versions, reason):
   return unittest.skipUnless(sys.version_info[:2] in versions, reason)
+
+
+def skipBeforePy(version, reason):
+  return unittest.skipIf(sys.version_info[:2] < version, reason)
+
+
+def skipFromPy(version, reason):
+  return unittest.skipUnless(sys.version_info[:2] < version, reason)
 # pylint: enable=invalid-name
diff --git a/pytype/tools/traces/traces_test.py b/pytype/tools/traces/traces_test.py
--- a/pytype/tools/traces/traces_test.py
+++ b/pytype/tools/traces/traces_test.py
@@ -215,13 +215,13 @@ def f(self, x):
         ((4, 0), "CALL_FUNCTION", "Foo", ("Type[Foo]", "Foo")),
         ((4, 0), call_method_op, "f", ("Callable[[Any], Any]", "int"))])
 
-  @test_utils.skipUnlessPy(
-      (3, 5), (3, 6), reason="instance methods match CALL_FUNCTION pre-3.7")
+  @test_utils.skipFromPy((3, 7),
+                         reason="instance methods match CALL_FUNCTION pre-3.7")
   def test_chain_pre37(self):
     self._test_chain("CALL_FUNCTION")
 
-  @test_utils.skipIfPy((3, 5), (3, 6),
-                       reason="instance methods match CALL_METHOD in 3.7+")
+  @test_utils.skipBeforePy((3, 7),
+                           reason="instance methods match CALL_METHOD in 3.7+")
   def test_chain_37(self):
     self._test_chain("CALL_METHOD")
 
@@ -255,13 +255,13 @@ def _test_literal(self, call_method_op):
     self.assertTracesEqual(matches, [
         ((1, 0), call_method_op, "upper", ("Callable[[], str]", "str"))])
 
-  @test_utils.skipUnlessPy(
-      (3, 5), (3, 6), reason="instance methods match CALL_FUNCTION pre-3.7")
+  @test_utils.skipFromPy((3, 7),
+                         reason="instance methods match CALL_FUNCTION pre-3.7")
   def test_literal_pre37(self):
     self._test_literal("CALL_FUNCTION")
 
-  @test_utils.skipIfPy((3, 5), (3, 6),
-                       reason="instance methods match CALL_METHOD in 3.7+")
+  @test_utils.skipBeforePy((3, 7),
+                           reason="instance methods match CALL_METHOD in 3.7+")
   def test_literal_37(self):
     self._test_literal("CALL_METHOD")