Repository: pypa/pip

Issue:
Pip fails with tar.bz2 even though there are other formats available
This bug/request is environment related.

Steps to reproduce:
1) pip install feedparser
File "/usr/lib/python2.7/tarfile.py", line 1742, in bz2open
    raise CompressionError("bz2 module is not available")
CompressionError: bz2 module is not available
2) pip install http://feedparser.googlecode.com/files/feedparser-5.1.1.tar.gz
works fine

Comments:
It would be nice if pip checks what tarball formats are available to download and in case of failure retry with other available formats. 
We have big project with lots of dependencies specified using setuptools and install_requires parameter. Automatic deployment doesn't work for us because pip fails installing bz2 archive. It's a pity because other tarball formats are available to download.
Maybe it would be better to favor tar.gz than tar.bz2?



Discussion:
It's easy to try importing bz2 once at startup and see if its there. I'd certainly accept a pull request to conditionally remove `.tar.bz2` from the list of valid sdist formats to look for if the bz2 module is not available (the user should get a stderr warning, though).

+1


Resolving patch:
diff --git a/pip/download.py b/pip/download.py
--- a/pip/download.py
+++ b/pip/download.py
@@ -23,7 +23,7 @@
 from pip.models import PyPI
 from pip.utils import (splitext, rmtree, format_size, display_path,
                        backup_dir, ask_path_exists, unpack_file,
-                       call_subprocess)
+                       call_subprocess, ARCHIVE_EXTENSIONS)
 from pip.utils.filesystem import check_path_owner
 from pip.utils.logging import indent_log
 from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner
@@ -453,11 +453,8 @@ def path_to_url(path):
 
 def is_archive_file(name):
     """Return True if `name` is a considered as an archive file."""
-    archives = (
-        '.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.whl'
-    )
     ext = splitext(name)[1].lower()
-    if ext in archives:
+    if ext in ARCHIVE_EXTENSIONS:
         return True
     return False
 
diff --git a/pip/index.py b/pip/index.py
--- a/pip/index.py
+++ b/pip/index.py
@@ -15,7 +15,8 @@
 
 from pip.compat import ipaddress
 from pip.utils import (
-    Inf, cached_property, normalize_name, splitext, normalize_path)
+    Inf, cached_property, normalize_name, splitext, normalize_path,
+    ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS)
 from pip.utils.deprecation import RemovedInPip8Warning
 from pip.utils.logging import indent_log
 from pip.exceptions import (
@@ -602,7 +603,6 @@ def _get_pages(self, locations, project_name):
 
                 all_locations.append(link)
 
-    _egg_fragment_re = re.compile(r'#egg=([^&]*)')
     _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)
     _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$')
 
@@ -628,11 +628,10 @@ def _package_versions(self, links, search_name):
             if v is not None:
                 yield v
 
-    def _known_extensions(self):
-        extensions = ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip')
-        if self.use_wheel:
-            return extensions + (wheel_ext,)
-        return extensions
+    def _log_skipped_link(self, link, reason):
+        if link not in self.logged_links:
+            logger.debug('Skipping link %s; %s', link, reason)
+            self.logged_links.add(link)
 
     def _link_package_versions(self, link, search_name):
         """Return an InstallationCandidate or None"""
@@ -644,51 +643,32 @@ def _link_package_versions(self, link, search_name):
         else:
             egg_info, ext = link.splitext()
             if not ext:
-                if link not in self.logged_links:
-                    logger.debug('Skipping link %s; not a file', link)
-                    self.logged_links.add(link)
+                self._log_skipped_link(link, 'not a file')
                 return
-            if egg_info.endswith('.tar'):
-                # Special double-extension case:
-                egg_info = egg_info[:-4]
-                ext = '.tar' + ext
-            if ext not in self._known_extensions():
-                if link not in self.logged_links:
-                    logger.debug(
-                        'Skipping link %s; unknown archive format: %s',
-                        link,
-                        ext,
-                    )
-                    self.logged_links.add(link)
+            if ext not in SUPPORTED_EXTENSIONS:
+                self._log_skipped_link(
+                    link, 'unsupported archive format: %s' % ext)
+                return
+            if not self.use_wheel and ext == wheel_ext:
+                self._log_skipped_link(link, '--no-use-wheel used')
                 return
             if "macosx10" in link.path and ext == '.zip':
-                if link not in self.logged_links:
-                    logger.debug('Skipping link %s; macosx10 one', link)
-                    self.logged_links.add(link)
+                self._log_skipped_link(link, 'macosx10 one')
                 return
             if ext == wheel_ext:
                 try:
                     wheel = Wheel(link.filename)
                 except InvalidWheelFilename:
-                    logger.debug(
-                        'Skipping %s because the wheel filename is invalid',
-                        link
-                    )
+                    self._log_skipped_link(link, 'invalid wheel filename')
                     return
                 if (pkg_resources.safe_name(wheel.name).lower() !=
                         pkg_resources.safe_name(search_name).lower()):
-                    logger.debug(
-                        'Skipping link %s; wrong project name (not %s)',
-                        link,
-                        search_name,
-                    )
+                    self._log_skipped_link(
+                        link, 'wrong project name (not %s)' % search_name)
                     return
                 if not wheel.supported():
-                    logger.debug(
-                        'Skipping %s because it is not compatible with this '
-                        'Python',
-                        link,
-                    )
+                    self._log_skipped_link(
+                        link, 'it is not compatible with this Python')
                     return
                 # This is a dirty hack to prevent installing Binary Wheels from
                 # PyPI unless it is a Windows or Mac Binary Wheel. This is
@@ -708,10 +688,10 @@ def _link_package_versions(self, link, search_name):
                             comes_from.url
                         ).netloc.endswith(PyPI.netloc)):
                     if not wheel.supported(tags=supported_tags_noarch):
-                        logger.debug(
-                            "Skipping %s because it is a pypi-hosted binary "
-                            "Wheel on an unsupported platform",
+                        self._log_skipped_link(
                             link,
+                            "it is a pypi-hosted binary "
+                            "Wheel on an unsupported platform",
                         )
                         return
                 version = wheel.version
@@ -719,11 +699,8 @@ def _link_package_versions(self, link, search_name):
         if not version:
             version = self._egg_info_matches(egg_info, search_name, link)
         if version is None:
-            logger.debug(
-                'Skipping link %s; wrong project name (not %s)',
-                link,
-                search_name,
-            )
+            self._log_skipped_link(
+                link, 'wrong project name (not %s)' % search_name)
             return
 
         if (link.internal is not None and not
@@ -733,7 +710,7 @@ def _link_package_versions(self, link, search_name):
                 self.allow_all_external):
             # We have a link that we are sure is external, so we should skip
             #   it unless we are allowing externals
-            logger.debug("Skipping %s because it is externally hosted.", link)
+            self._log_skipped_link(link, 'it is externally hosted')
             self.need_warn_external = True
             return
 
@@ -744,10 +721,8 @@ def _link_package_versions(self, link, search_name):
             # We have a link that we are sure we cannot verify its integrity,
             #   so we should skip it unless we are allowing unsafe installs
             #   for this requirement.
-            logger.debug(
-                "Skipping %s because it is an insecure and unverifiable file.",
-                link,
-            )
+            self._log_skipped_link(
+                link, 'it is an insecure and unverifiable file')
             self.need_warn_unverified = True
             return
 
@@ -756,9 +731,8 @@ def _link_package_versions(self, link, search_name):
             version = version[:match.start()]
             py_version = match.group(1)
             if py_version != sys.version[:3]:
-                logger.debug(
-                    'Skipping %s because Python version is incorrect', link
-                )
+                self._log_skipped_link(
+                    link, 'Python version is incorrect')
                 return
         logger.debug('Found link %s, version: %s', link, version)
 
@@ -828,7 +802,7 @@ def get_page(cls, link, skip_archives=True, session=None):
         try:
             if skip_archives:
                 filename = link.filename
-                for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']:
+                for bad_ext in ARCHIVE_EXTENSIONS:
                     if filename.endswith(bad_ext):
                         content_type = cls._get_content_type(
                             url, session=session,
diff --git a/pip/utils/__init__.py b/pip/utils/__init__.py
--- a/pip/utils/__init__.py
+++ b/pip/utils/__init__.py
@@ -38,12 +38,25 @@
            'make_path_relative', 'normalize_path',
            'renames', 'get_terminal_size', 'get_prog',
            'unzip_file', 'untar_file', 'unpack_file', 'call_subprocess',
-           'captured_stdout', 'remove_tracebacks']
+           'captured_stdout', 'remove_tracebacks',
+           'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS']
 
 
 logger = logging.getLogger(__name__)
 
 
+BZ2_EXTENSIONS = ('.tar.bz2', '.tbz')
+ZIP_EXTENSIONS = ('.zip', '.whl')
+TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar')
+ARCHIVE_EXTENSIONS = ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS
+try:
+    import bz2  # noqa
+    SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS
+except ImportError:
+    logger.debug('bz2 module is not available')
+    SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS
+
+
 def import_or_raise(pkg_or_module_string, ExceptionType, *args, **kwargs):
     try:
         return __import__(pkg_or_module_string)
@@ -552,8 +565,7 @@ def untar_file(filename, location):
         os.makedirs(location)
     if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'):
         mode = 'r:gz'
-    elif (filename.lower().endswith('.bz2') or
-            filename.lower().endswith('.tbz')):
+    elif filename.lower().endswith(BZ2_EXTENSIONS):
         mode = 'r:bz2'
     elif filename.lower().endswith('.tar'):
         mode = 'r'
@@ -621,8 +633,7 @@ def untar_file(filename, location):
 def unpack_file(filename, location, content_type, link):
     filename = os.path.realpath(filename)
     if (content_type == 'application/zip' or
-            filename.endswith('.zip') or
-            filename.endswith('.whl') or
+            filename.lower().endswith(ZIP_EXTENSIONS) or
             zipfile.is_zipfile(filename)):
         unzip_file(
             filename,
@@ -631,8 +642,7 @@ def unpack_file(filename, location, content_type, link):
         )
     elif (content_type == 'application/x-gzip' or
             tarfile.is_tarfile(filename) or
-            splitext(filename)[1].lower() in (
-                '.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')):
+            filename.lower().endswith(TAR_EXTENSIONS + BZ2_EXTENSIONS)):
         untar_file(filename, location)
     elif (content_type and content_type.startswith('text/html') and
             is_svn_page(file_contents(filename))):


Test patch:
diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py
--- a/tests/unit/test_finder.py
+++ b/tests/unit/test_finder.py
@@ -134,7 +134,7 @@ def test_skip_invalid_wheel_link(self, caplog, data):
             finder.find_requirement(req, True)
 
         assert (
-            "invalid.whl because the wheel filename is invalid"
+            "invalid.whl; invalid wheel filename"
             in caplog.text()
         )