Repository: pypa/pip

Issue:
Support .tar.xz packages
Hi,

the following patch enables detection and unpacking of .tar.xz archives. Since xz (ie. the lzma compression method) is not supported by the tarfile module, the code falls back to system binary. Not pretty, but works for me - tested with "pip install linkchecker".

See also http://bugs.python.org/issue6715 for the Python bug adding .xz support (but only in Python >= 3.3).

There is apparently no way to attach diff files, so here goes:

``` diff
--- download.py 2012-11-13 15:10:22.086133699 +0100
+++ /usr/lib/python2.7/dist-packages/pip/download.py    2012-11-13 15:10:34.189942352 +0100
@@ -279,7 +279,7 @@

 def is_archive_file(name):
     """Return True if `name` is a considered as an archive file."""
-    archives = ('.zip', '.tar.gz', '.tar.bz2', '.tgz', '.tar', '.pybundle')
+    archives = ('.zip', '.tar.gz', '.tar.bz2', '.tar.xz', '.tgz', '.tar', '.pybundle')
     ext = splitext(name)[1].lower()
     if ext in archives:
         return True
--- index.py    2012-11-13 15:08:10.936207032 +0100
+++ /usr/lib/python2.7/dist-packages/pip/index.py   2012-11-13 15:09:48.102670938 +0100
@@ -292,7 +292,7 @@
                 # Special double-extension case:
                 egg_info = egg_info[:-4]
                 ext = '.tar' + ext
-            if ext not in ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip'):
+            if ext not in ('.tar.gz', '.tar.bz2', '.tar.xz', '.tar', '.tgz', '.zip'):
                 if link not in self.logged_links:
                     logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext))
                     self.logged_links.add(link)
@@ -428,7 +428,7 @@
                     if cache.is_archive(url):
                         return None
                 filename = link.filename
-                for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']:
+                for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tar.xz', '.tgz', '.zip']:
                     if filename.endswith(bad_ext):
                         content_type = cls._get_content_type(url)
                         if content_type.lower().startswith('text/html'):
--- util.py 2012-11-13 15:10:44.565778323 +0100
+++ /usr/lib/python2.7/dist-packages/pip/util.py    2012-11-13 15:44:27.159051300 +0100
@@ -401,15 +401,26 @@
     finally:
         zipfp.close()

+def decompress_xz(filename):
+    import subprocess
+    subprocess.check_call(["xz", "-dk", filename])
+    return filename[:-3]

 def untar_file(filename, location):
     """Untar the file (tar file located at filename) to the destination location"""
+    remove_file = False
     if not os.path.exists(location):
         os.makedirs(location)
     if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'):
         mode = 'r:gz'
     elif filename.lower().endswith('.bz2') or filename.lower().endswith('.tbz'):
         mode = 'r:bz2'
+    elif filename.lower().endswith('.xz'):
+        mode = 'r'
+        # extract to (temporary) .tar file since the tarfile module does not
+        # support xz compression
+        filename = decompress_xz(filename)
+        remove_file = True
     elif filename.lower().endswith('.tar'):
         mode = 'r'
     else:
@@ -464,6 +475,8 @@
                 fp.close()
     finally:
         tar.close()
+        if remove_file:
+            os.unlink(filename)


 def create_download_cache_folder(folder):
@@ -490,7 +503,7 @@
         unzip_file(filename, location, flatten=not filename.endswith('.pybundle'))
     elif (content_type == 'application/x-gzip'
           or tarfile.is_tarfile(filename)
-          or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')):
+          or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tar.xz', '.tgz', '.tbz')):
         untar_file(filename, location)
     elif (content_type and content_type.startswith('text/html')
           and is_svn_page(file_contents(filename))):

```



Discussion:
Can you submit a pull request from your branch on github.


Resolving patch:
diff --git a/pip/utils/__init__.py b/pip/utils/__init__.py
--- a/pip/utils/__init__.py
+++ b/pip/utils/__init__.py
@@ -49,15 +49,24 @@
 logger = std_logging.getLogger(__name__)
 
 BZ2_EXTENSIONS = ('.tar.bz2', '.tbz')
+XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', '.tar.lz', '.tar.lzma')
 ZIP_EXTENSIONS = ('.zip', '.whl')
 TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar')
-ARCHIVE_EXTENSIONS = ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS
+ARCHIVE_EXTENSIONS = (
+    ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS)
+SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS
 try:
     import bz2  # noqa
-    SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS
+    SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS
 except ImportError:
     logger.debug('bz2 module is not available')
-    SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS
+
+try:
+    # Only for Python 3.3+
+    import lzma  # noqa
+    SUPPORTED_EXTENSIONS += XZ_EXTENSIONS
+except ImportError:
+    logger.debug('lzma module is not available')
 
 
 def import_or_raise(pkg_or_module_string, ExceptionType, *args, **kwargs):
@@ -516,6 +525,8 @@ def untar_file(filename, location):
         mode = 'r:gz'
     elif filename.lower().endswith(BZ2_EXTENSIONS):
         mode = 'r:bz2'
+    elif filename.lower().endswith(XZ_EXTENSIONS):
+        mode = 'r:xz'
     elif filename.lower().endswith('.tar'):
         mode = 'r'
     else:
@@ -589,7 +600,8 @@ def unpack_file(filename, location, content_type, link):
         )
     elif (content_type == 'application/x-gzip' or
             tarfile.is_tarfile(filename) or
-            filename.lower().endswith(TAR_EXTENSIONS + BZ2_EXTENSIONS)):
+            filename.lower().endswith(
+                TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS)):
         untar_file(filename, location)
     elif (content_type and content_type.startswith('text/html') and
             is_svn_page(file_contents(filename))):


Test patch:
diff --git a/tests/data/packages/singlemodule-0.0.1.tar.lzma b/tests/data/packages/singlemodule-0.0.1.tar.lzma
new file mode 100644
Binary files /dev/null and b/tests/data/packages/singlemodule-0.0.1.tar.lzma differ
diff --git a/tests/data/packages/singlemodule-0.0.1.tar.xz b/tests/data/packages/singlemodule-0.0.1.tar.xz
new file mode 100644
Binary files /dev/null and b/tests/data/packages/singlemodule-0.0.1.tar.xz differ
diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py
--- a/tests/functional/test_install.py
+++ b/tests/functional/test_install.py
@@ -914,3 +914,21 @@ def test_install_editable_with_wrong_egg_name(script):
             "for project name pkga. Fix your #egg=pkgb "
             "fragments.") in result.stderr
     assert "Successfully installed pkga" in str(result), str(result)
+
+
+def test_install_tar_xz(script, data):
+    try:
+        import lzma  # noqa
+    except ImportError:
+        pytest.skip("No lzma support")
+    res = script.pip('install', data.packages / 'singlemodule-0.0.1.tar.xz')
+    assert "Successfully installed singlemodule-0.0.1" in res.stdout, res
+
+
+def test_install_tar_lzma(script, data):
+    try:
+        import lzma  # noqa
+    except ImportError:
+        pytest.skip("No lzma support")
+    res = script.pip('install', data.packages / 'singlemodule-0.0.1.tar.lzma')
+    assert "Successfully installed singlemodule-0.0.1" in res.stdout, res