Repository: google/pytype

Issue:
Add partial support for nested classes in pyi files.
This PR will have to be imported and re-exported rather than merged directly.

It attempts to resolve https://github.com/google/pytype/issues/134 by fixing the second issue that prevented pytype from parsing the yaml pyi files, the existence of a nested class in yaml.constructor. I decided to change our parser rather than typeshed because I found more examples of nested classes in typeshed/third_party/2and3/google/protobuf/.

I'm not yet calling the issue fixed because I still want to do a release (hence the `__version__` change), verify the fix, and then change typeshed's pytype_test to cover yaml/ going forward.


Discussion:

Resolving patch:
diff --git a/pytype/__version__.py b/pytype/__version__.py
--- a/pytype/__version__.py
+++ b/pytype/__version__.py
@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2018.8.10'
+__version__ = '2018.9.7'
diff --git a/pytype/pyi/parser.py b/pytype/pyi/parser.py
--- a/pytype/pyi/parser.py
+++ b/pytype/pyi/parser.py
@@ -251,12 +251,12 @@ class _Parser(object):
     new_constant()
     add_alias_or_constant()
     add_import()
+    new_class()
     new_type()
     new_union_type()
     new_function()
     new_named_tuple()
     regiser_class_name()
-    add_class()
     add_type_var()
     if_begin()
     if_elif()
@@ -347,7 +347,6 @@ def __init__(self, version, platform):
     # final TypeDeclUnit.
     self._constants = []
     self._aliases = []
-    self._classes = []
     self._type_params = []
     self._module_path_map = {}
     self._generated_classes = collections.defaultdict(list)
@@ -416,7 +415,7 @@ def parse(self, src, name, filename):
   def _build_type_decl_unit(self, defs):
     """Return a pytd.TypeDeclUnit for the given defs (plus parser state)."""
     # defs contains both constant and function definitions.
-    constants, functions, aliases, slots = _split_definitions(defs)
+    constants, functions, aliases, slots, classes = _split_definitions(defs)
     assert not slots  # slots aren't allowed on the module level
     assert not aliases  # We handle top-level aliases in add_alias_or_constant.
     constants.extend(self._constants)
@@ -424,7 +423,7 @@ def _build_type_decl_unit(self, defs):
     generated_classes = [x for class_list in self._generated_classes.values()
                          for x in class_list]
 
-    classes = generated_classes + self._classes
+    classes = generated_classes + classes
     functions = _merge_method_signatures(functions)
 
     name_to_class = {c.name: c for c in classes}
@@ -934,8 +933,8 @@ def register_class_name(self, class_name):
       return
     self._type_map[class_name] = pytd.NamedType(class_name)
 
-  def add_class(self, class_name, parent_args, defs):
-    """Add a class to the module.
+  def new_class(self, class_name, parent_args, defs):
+    """Create a new class.
 
     Args:
       class_name: The name of the class (a string).
@@ -943,9 +942,12 @@ def add_class(self, class_name, parent_args, defs):
           Parent types must be instances of pytd.Type.  Keyword tuples must
           appear at the end of the list.  Currently the only supported keyword
           is 'metaclass'.
-      defs: A list of constant (pytd.Constant) and function (_NameAndSig)
-          definitions.
+      defs: A list of constant (pytd.Constant), function (_NameAndSig), alias
+          (pytd.Alias), slot (_SlotDecl), and class (pytd.Class) definitions.
 
+    Returns:
+      None if the class definition is inside a non-active conditional,
+      otherwise a new pytd.Class.
     Raises:
       ParseError: if defs contains duplicate names (excluding multiple
           definitions of a function, which is allowed).
@@ -964,7 +966,10 @@ def add_class(self, class_name, parent_args, defs):
           raise ParseError("Only 'metaclass' allowed as classdef kwarg")
         metaclass = value
 
-    constants, methods, aliases, slots = _split_definitions(defs)
+    constants, methods, aliases, slots, classes = _split_definitions(defs)
+    # TODO(rechen): retain more information about nested classes.
+    for cls in classes:
+      constants.append(self.new_constant(cls.name, pytd.NamedType("type")))
 
     all_names = (list(set(f.name for f in methods)) +
                  [c.name for c in constants] +
@@ -979,6 +984,8 @@ def add_class(self, class_name, parent_args, defs):
     # This check is performed after the above error checking so that errors
     # will be spotted even in non-active conditional code.
     if not self._current_condition.active:
+      # Returning early is safe because if_end() takes care of discarding
+      # definitions inside non-active conditions.
       return
 
     if aliases:
@@ -1004,13 +1011,12 @@ def add_class(self, class_name, parent_args, defs):
     # Ensure that old style classes inherit from classobj.
     if not parents and class_name not in ["classobj", "object"]:
       parents = (pytd.NamedType("classobj"),)
-    cls = pytd.Class(name=class_name, metaclass=metaclass,
-                     parents=tuple(parents),
-                     methods=tuple(methods),
-                     constants=tuple(constants),
-                     slots=slots,
-                     template=())
-    self._classes.append(cls)
+    return pytd.Class(name=class_name, metaclass=metaclass,
+                      parents=tuple(parents),
+                      methods=tuple(methods),
+                      constants=tuple(constants),
+                      slots=slots,
+                      template=())
 
   def add_type_var(self, name, name_arg, args):
     """Add a type variable, <name> = TypeVar(<name_arg>, <args>)."""
@@ -1259,6 +1265,7 @@ def _split_definitions(defs):
   functions = []
   aliases = []
   slots = None
+  classes = []
   for d in defs:
     if isinstance(d, pytd.Constant):
       if d.name == "__slots__":
@@ -1278,9 +1285,11 @@ def _split_definitions(defs):
         raise ParseError("Entries in __slots__ can only be strings")
       slots = tuple(p.name for p in d.slots.parameters
                     if isinstance(p, pytd.NamedType))
+    elif isinstance(d, pytd.Class):
+      classes.append(d)
     else:
       raise TypeError("Unexpected definition type %s" % type(d))
-  return constants, functions, aliases, slots
+  return constants, functions, aliases, slots, classes
 
 
 def _is_int_tuple(value):


Test patch:
diff --git a/pytype/pyi/parser_test.py b/pytype/pyi/parser_test.py
--- a/pytype/pyi/parser_test.py
+++ b/pytype/pyi/parser_test.py
@@ -232,6 +232,15 @@ class A:
           __slots__ = ["foo", ?]
     """, 1, "Entries in __slots__ can only be strings")
 
+  def test_nested_class(self):
+    self.check("""\
+      class A:
+          class B: ...
+    """, """\
+      class A:
+          B = ...  # type: type
+    """)
+
   def test_import(self):
     self.check("import foo.bar.baz", "")
     self.check("import a as b")
@@ -1301,11 +1310,14 @@ class Foo:
     """, 1, "Illegal value for alias 'a'")
 
   def test_no_class(self):
-    self.check_error("""\
+    self.check("""\
       class Foo:
-        if sys.version_info > (2, 7, 0):
+        if sys.version_info <= (2, 7, 0):
           class Bar: ...
-    """, 3, "syntax error")
+    """, """\
+      class Foo:
+          pass
+    """)
 
   def test_no_typevar(self):
     self.check_error("""\
diff --git a/pytype/tools/xref/indexer_test.py b/pytype/tools/xref/indexer_test.py
--- a/pytype/tools/xref/indexer_test.py
+++ b/pytype/tools/xref/indexer_test.py
@@ -112,34 +112,6 @@ def test_resolved_imports(self):
       self.assertEqual(ix.modules["module.c"], "a.b")
       self.assertEqual(ix.modules["module.r"], "p.q")
 
-  def test_import_ref(self):
-    # We need all imports to be valid for pytype
-    code = """\
-        import foo
-        x = foo.Bar
-    """
-    stub = "class Bar: pass"
-    with file_utils.Tempdir() as d:
-      d.create_file("t.py", code)
-      d.create_file("foo.pyi", stub)
-      options = config.Options([d["t.py"]])
-      options.tweak(pythonpath=[d.path], version=self.python_version)
-      ix = indexer.process_file(options)
-      self.assertDef(ix, "module.foo", "foo", "Import")
-      self.assertDef(ix, "module.x", "x", "Store")
-      self.assertEqual(ix.modules["module.foo"], "foo")
-      expected = [
-          ({"name": "foo", "typ": "Name", "scope": "module",
-            "location": (2, 4)},
-           {"name": "foo", "scope": "module"}),
-          ({"name": "foo.Bar", "typ": "Attribute", "scope": "module",
-            "location": (2, 4)},
-           {"name": "Bar", "scope": "foo/module"})
-      ]
-      for (ref, defn), (expected_ref, expected_defn) in zip(ix.links, expected):
-        self.assertAttrs(ref, expected_ref)
-        self.assertAttrs(defn, expected_defn)
-
   def test_docstrings(self):
     ix = self.index_code("""\
         class A():