Repository: pypa/pip

Issue:
vendor: Don't discard traces when an ImportError occurs
In Homebrew, we ran into [an issue](https://github.com/Homebrew/homebrew/issues/40516) where an ImportError raised when pip failed to import a module from within `_vendor`. The trace was unhelpful, and after a bit of digging I noticed that the original trace is being discarded and a new `ImportError` raised that masks the actual location of the exception.

For example, the provided trace looked like this:

```
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/site-packages/pip-7.0.1-py2.7.egg/pip/__init__.py", line 15, in <module>
    from pip.vcs import git, mercurial, subversion, bazaar  # noqa
  File "/usr/local/lib/python2.7/site-packages/pip-7.0.1-py2.7.egg/pip/vcs/subversion.py", line 9, in <module>
    from pip.index import Link
  File "/usr/local/lib/python2.7/site-packages/pip-7.0.1-py2.7.egg/pip/index.py", line 30, in <module>
    from pip.wheel import Wheel, wheel_ext
  File "/usr/local/lib/python2.7/site-packages/pip-7.0.1-py2.7.egg/pip/wheel.py", line 34, in <module>
    from pip._vendor.distlib.scripts import ScriptMaker
  File "/usr/local/lib/python2.7/site-packages/pip-7.0.1-py2.7.egg/pip/_vendor/__init__.py", line 92, in load_module
    raise ImportError("No module named '%s'" % (name,))
ImportError: No module named 'pip._vendor.distlib.scripts'
```

However, the actual `ImportError` was being raised [here](https://github.com/pypa/pip/blob/develop/pip/_vendor/distlib/compat.py#L31-L34) (Python was built without `_ssl`, and so one of the constants is unavailable).

I think it would make debugging easier to let the original `ImportError` propagate.



Discussion:

Resolving patch:
diff --git a/pip/_vendor/__init__.py b/pip/_vendor/__init__.py
--- a/pip/_vendor/__init__.py
+++ b/pip/_vendor/__init__.py
@@ -11,6 +11,10 @@
 import os.path
 import sys
 
+# Downstream redistributors which have debundled our dependencies should also
+# patch this value to be true. This will trigger the additional patching
+# to cause things like "six" to be available as pip.
+DEBUNDLED = False
 
 # By default, look in this directory for a bunch of .whl files which we will
 # add to the beginning of sys.path before attempting to import anything. This
@@ -18,91 +22,45 @@
 # wish to create their own Wheels for our dependencies to aid in debundling.
 WHEEL_DIR = os.path.abspath(os.path.dirname(__file__))
 
-# Actually look inside of WHEEL_DIR to find .whl files and add them to the
-# front of our sys.path.
-sys.path[:] = glob.glob(os.path.join(WHEEL_DIR, "*.whl")) + sys.path
 
-
-class VendorAlias(object):
-
-    def __init__(self, package_names):
-        self._package_names = package_names
-        self._vendor_name = __name__
-        self._vendor_pkg = self._vendor_name + "."
-        self._vendor_pkgs = [
-            self._vendor_pkg + name for name in self._package_names
-        ]
-
-    def find_module(self, fullname, path=None):
-        if fullname.startswith(self._vendor_pkg):
-            return self
-
-    def load_module(self, name):
-        # Ensure that this only works for the vendored name
-        if not name.startswith(self._vendor_pkg):
-            raise ImportError(
-                "Cannot import %s, must be a subpackage of '%s'." % (
-                    name, self._vendor_name,
-                )
-            )
-        if not (name == self._vendor_name or
-                any(name.startswith(pkg) for pkg in self._vendor_pkgs)):
-            raise ImportError(
-                "Cannot import %s, must be one of %s." % (
-                    name, self._vendor_pkgs
-                )
-            )
-
-        # Check to see if we already have this item in sys.modules, if we do
-        # then simply return that.
-        if name in sys.modules:
-            return sys.modules[name]
-
-        # Check to see if we can import the vendor name
-        try:
-            # We do this dance here because we want to try and import this
-            # module without hitting a recursion error because of a bunch of
-            # VendorAlias instances on sys.meta_path
-            real_meta_path = sys.meta_path[:]
-            try:
-                sys.meta_path = [
-                    m for m in sys.meta_path
-                    if not isinstance(m, VendorAlias)
-                ]
-                __import__(name)
-                module = sys.modules[name]
-            finally:
-                # Re-add any additions to sys.meta_path that were made while
-                # during the import we just did, otherwise things like
-                # pip._vendor.six.moves will fail.
-                for m in sys.meta_path:
-                    if m not in real_meta_path:
-                        real_meta_path.append(m)
-
-                # Restore sys.meta_path with any new items.
-                sys.meta_path = real_meta_path
-        except ImportError:
-            # We can't import the vendor name, so we'll try to import the
-            # "real" name.
-            real_name = name[len(self._vendor_pkg):]
-            try:
-                __import__(real_name)
-                module = sys.modules[real_name]
-            except ImportError:
-                raise ImportError("No module named '%s'" % (name,))
-
-        # If we've gotten here we've found the module we're looking for, either
-        # as part of our vendored package, or as the real name, so we'll add
-        # it to sys.modules as the vendored name so that we don't have to do
-        # the lookup again.
-        sys.modules[name] = module
-
-        # Finally, return the loaded module
-        return module
-
-
-sys.meta_path.append(VendorAlias([
-    "_markerlib", "cachecontrol", "certifi", "colorama", "distlib", "html5lib",
-    "ipaddress", "lockfile", "packaging", "pkg_resources", "progress",
-    "requests", "retrying", "six",
-]))
+# Define a small helper function to alias our vendored modules to the real ones
+# if the vendored ones do not exist. This idea of this was taken from
+# https://github.com/kennethreitz/requests/pull/2567.
+def vendored(modulename):
+    vendored_name = "{0}.{1}".format(__name__, modulename)
+
+    try:
+        __import__(vendored_name, globals(), locals(), level=0)
+    except ImportError:
+        __import__(modulename, globals(), locals(), level=0)
+        sys.modules[vendored_name] = sys.modules[modulename]
+        base, head = vendored_name.rsplit(".", 1)
+        setattr(sys.modules[base], head, sys.modules[modulename])
+
+
+# If we're operating in a debundled setup, then we want to go ahead and trigger
+# the aliasing of our vendored libraries as well as looking for wheels to add
+# to our sys.path. This will cause all of this code to be a no-op typically
+# however downstream redistributors can enable it in a consistent way across
+# all platforms.
+if DEBUNDLED:
+    # Actually look inside of WHEEL_DIR to find .whl files and add them to the
+    # front of our sys.path.
+    sys.path[:] = glob.glob(os.path.join(WHEEL_DIR, "*.whl")) + sys.path
+
+    # Actually alias all of our vendored dependencies.
+    vendored("cachecontrol")
+    vendored("colorama")
+    vendored("distlib")
+    vendored("html5lib")
+    vendored("lockfile")
+    vendored("six")
+    vendored("six.moves")
+    vendored("six.moves.urllib")
+    vendored("packaging")
+    vendored("packaging.version")
+    vendored("packaging.specifiers")
+    vendored("pkg_resources")
+    vendored("progress")
+    vendored("retrying")
+    vendored("requests")
diff --git a/pip/index.py b/pip/index.py
--- a/pip/index.py
+++ b/pip/index.py
@@ -484,11 +484,18 @@ def find_requirement(self, req, upgrade):
         May raise DistributionNotFound or BestVersionAlreadyInstalled
         """
         all_versions = self._find_all_versions(req.name)
-        # Filter out anything which doesn't match our specifier
 
+        # Filter out anything which doesn't match our specifier
         _versions = set(
             req.specifier.filter(
-                [x.version for x in all_versions],
+                # We turn the version object into a str here because otherwise
+                # when we're debundled but setuptools isn't, Python will see
+                # packaging.version.Version and
+                # pkg_resources._vendor.packaging.version.Version as different
+                # types. This way we'll use a str as a common data interchange
+                # format. If we stop using the pkg_resources provided specifier
+                # and start using our own, we can drop the cast to str().
+                [str(x.version) for x in all_versions],
                 prereleases=(
                     self.allow_all_prereleases
                     if self.allow_all_prereleases else None
@@ -496,7 +503,8 @@ def find_requirement(self, req, upgrade):
             )
         )
         applicable_versions = [
-            x for x in all_versions if x.version in _versions
+            # Again, converting to str to deal with debundling.
+            x for x in all_versions if str(x.version) in _versions
         ]
 
         if req.satisfied_by is not None:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -7,12 +7,6 @@
 from setuptools.command.test import test as TestCommand
 
 
-# We support this primarily for downstream distributions. This is NOT intended
-# to be used by end users and any actual use of thus is not considered a
-# supported configuration.
-NO_VENDOR = ("PIP_NO_VENDOR_FOR_DOWNSTREAM" in os.environ)
-
-
 here = os.path.abspath(os.path.dirname(__file__))
 
 
@@ -49,24 +43,6 @@ def find_version(*file_paths):
 
 tests_require = ['pytest', 'virtualenv>=1.10', 'scripttest>=1.3', 'mock']
 
-find_excludes = ["contrib", "docs", "tests*", "tasks"]
-
-py_modules = []
-
-package_data = {}
-
-
-if NO_VENDOR:
-    find_excludes += ["pip._vendor", "pip._vendor.*"]
-    py_modules += ["pip._vendor.__init__"]
-else:
-    package_data.update({
-        "pip._vendor.certifi": ["*.pem"],
-        "pip._vendor.requests": ["*.pem"],
-        "pip._vendor.distlib._backport": ["sysconfig.cfg"],
-        "pip._vendor.distlib": ["t32.exe", "t64.exe", "w32.exe", "w64.exe"],
-    })
-
 
 setup(
     name="pip",
@@ -92,9 +68,13 @@ def find_version(*file_paths):
     author_email='python-virtualenv@groups.google.com',
     url='https://pip.pypa.io/',
     license='MIT',
-    py_modules=py_modules,
-    packages=find_packages(exclude=find_excludes),
-    package_data=package_data,
+    packages=find_packages(exclude=["contrib", "docs", "tests*", "tasks"]),
+    package_data={
+        "pip._vendor.certifi": ["*.pem"],
+        "pip._vendor.requests": ["*.pem"],
+        "pip._vendor.distlib._backport": ["sysconfig.cfg"],
+        "pip._vendor.distlib": ["t32.exe", "t64.exe", "w32.exe", "w64.exe"],
+    },
     entry_points={
         "console_scripts": [
             "pip=pip:main",
@@ -109,18 +89,3 @@ def find_version(*file_paths):
     },
     cmdclass={'test': PyTest},
 )
-
-
-if NO_VENDOR:
-    print("""
-###########################################################################
-## IMPORTANT IMPORTANT IMPORTANT IMPORTANT IMPORTANT IMPORTANT IMPORTANT ##
-###########################################################################
-## You are installing pip without the bundled dependencies.              ##
-##                                                                       ##
-## This is not an officially supported option for end users and should   ##
-## only be used by downstream redistributors such as the various Linux   ##
-## distributions. This method of installation is not as well tested by   ##
-## the pip team.                                                         ##
-###########################################################################
-""")


Test patch: