Repository: google/pytype

Issue:
fix handling of stdlib/@python2
Related to python/typeshed#5329: we will now set 2.7 as the minimum version in VERSIONS for libraries that existed in 2.7. I tested this by running pytype_test.py in typeshed.

Happily, fixing this also helped me find out that I got a few versions wrong in the VERSIONS file.


Discussion:

Resolving patch:
diff --git a/pytype/__version__.py b/pytype/__version__.py
--- a/pytype/__version__.py
+++ b/pytype/__version__.py
@@ -1,2 +1,2 @@
 # pylint: skip-file
-__version__ = '2021.04.26'
+__version__ = '2021.05.04'
diff --git a/pytype/pytd/typeshed.py b/pytype/pytd/typeshed.py
--- a/pytype/pytd/typeshed.py
+++ b/pytype/pytd/typeshed.py
@@ -105,14 +105,16 @@ def _load_stdlib_versions(self):
 
     VERSIONS lists the stdlib modules with the Python version in which they were
     first added, in the format `{module}: {min_major}.{min_minor}-` or
-    `{module}: {min_major}.{min_minor}-{max_major}.{max_minor}`. Note that this
-    file ignores the stdlib/@python2 subdirectory! If stdlib/foo.pyi targets
-    Python 3.6+ and stdlib/@python2/foo.pyi, 2.7, VERSIONS will contain
-    `foo: 3.6-`.
+    `{module}: {min_major}.{min_minor}-{max_major}.{max_minor}`.
 
     Returns:
       A mapping from module name to ((min_major, min_minor),
-      (max_major, max_minor)) Python version. The max tuple can be `None`.
+      (max_major, max_minor), use_python2) Python version.
+      The max tuple can be `None`. The use_python2 member indicates
+      whether stubs are present in the @python2 directory. If so,
+      stubs outside @python2 should not be used. This is relevant
+      if a package contains more files in Python 3 than it did in
+      Python 2.
     """
     _, text = self._load_file(os.path.join("stdlib", "VERSIONS"))
     versions = {}
@@ -127,7 +129,10 @@ def _load_stdlib_versions(self):
       maximum = ((int(max_major), int(max_minor))
                  if max_major is not None and max_minor is not None
                  else None)
-      versions[module] = minimum, maximum
+      use_python2 = min_major == "2" and (
+          self._file_exists(os.path.join("stdlib", "@python2", module + ".pyi"))
+          or self._file_exists(os.path.join("stdlib", "@python2", module)))
+      versions[module] = minimum, maximum, use_python2
     return versions
 
   def _load_third_party_packages(self):
@@ -136,8 +141,8 @@ def _load_third_party_packages(self):
     stubs/ contains type information for third-party packages. Each top-level
     directory corresponds to one PyPI package and contains one or more modules,
     plus a metadata file (METADATA.toml). If there are separate Python 2 stubs,
-    they live in an @python2 subdirectory. Unlike stdlib/VERSIONS, METADATA.toml
-    does take @python2 into account, so if a package has both foo.pyi and
+    they live in an @python2 subdirectory. METADATA.toml
+    takes @python2 into account, so if a package has both foo.pyi and
     @python2/foo.pyi, METADATA.toml will contain `python2 = True`.
 
     Returns:
@@ -211,14 +216,14 @@ def get_module_file(self, toplevel, module, version):
     if toplevel == "stdlib":
       # stubs for the stdlib 'foo' module are located in either stdlib/foo or
       # (for Python 2) stdlib/@python2/foo. The VERSIONS file tells us whether
-      # stdlib/foo exists and what versions it targets; we also have to
-      # separately check for stdlib/@python2/foo.
+      # stdlib/foo exists and what versions it targets; we always have to check
+      # @python2 first for Python 2 stubs.
       path = os.path.join(toplevel, module_path)
+      if version[0] == 2:
+        paths.append(os.path.join(toplevel, "@python2", module_path))
       if (self._is_module_in_typeshed(module_parts[0], version) or
           path in self.missing):
         paths.append(path)
-      if version[0] == 2:
-        paths.append(os.path.join(toplevel, "@python2", module_path))
     elif toplevel == "third_party":
       # For third-party modules, we grab the alphabetically first package that
       # provides a module with the specified name in the right version.
@@ -251,16 +256,17 @@ def get_module_file(self, toplevel, module, version):
   def _is_module_in_typeshed(self, name, version):
     if name not in self._stdlib_versions:
       return False
-    min_version, max_version = self._stdlib_versions[name]
+    min_version, max_version, _ = self._stdlib_versions[name]
     return (min_version <= version and
             (max_version is None or max_version >= version))
 
   def get_typeshed_paths(self, python_version):
     """Gets the paths to typeshed's version-specific pyi files."""
     major, _ = python_version
-    typeshed_subdirs = ["stdlib"]
+    typeshed_subdirs = []
     if major == 2:
       typeshed_subdirs.append(os.path.join("stdlib", "@python2"))
+    typeshed_subdirs.append("stdlib")
     for packages in self._third_party_packages.values():
       for package, v in packages:
         if v == major:
@@ -355,7 +361,10 @@ def get_python_major_versions(self, filename):
       return (2,)
     parts = filename.split(os.path.sep)
     if parts[0] == "stdlib":
-      if self._stdlib_versions[os.path.splitext(parts[1])[0]][0] >= (3, 0):
+      min_version, _, use_python2 = self._stdlib_versions[
+          os.path.splitext(parts[1])[0]]
+      # If use_python2 is true, we just use the stubs in @python2
+      if min_version >= (3, 0) or use_python2:
         return (3,)
       else:
         return (2, 3)


Test patch:
diff --git a/pytype/pytd/typeshed_test.py b/pytype/pytd/typeshed_test.py
--- a/pytype/pytd/typeshed_test.py
+++ b/pytype/pytd/typeshed_test.py
@@ -90,7 +90,7 @@ def test_blacklisted_modules(self):
   def test_carriage_return(self):
     # _env_home is used in preference to _root, so make sure it's unset.
     self.ts._env_home = None
-    self.ts._stdlib_versions["foo"] = ((3, 8), None)
+    self.ts._stdlib_versions["foo"] = ((3, 8), None, False)
     with file_utils.Tempdir() as d:
       d.create_file("stdlib/foo.pyi", b"x: int\r\n")
       self.ts._root = d.path
@@ -98,7 +98,7 @@ def test_carriage_return(self):
     self.assertEqual(src, "x: int\n")
 
   def test_carriage_return_custom_root(self):
-    self.ts._stdlib_versions["foo"] = ((3, 8), None)
+    self.ts._stdlib_versions["foo"] = ((3, 8), None, False)
     with file_utils.Tempdir() as d:
       d.create_file("stdlib/foo.pyi", b"x: int\r\n")
       self.ts._env_home = d.path