Repository: pypa/pip

Issue:
When a package from the download cache fails the hash check, pip should purge that file from the cache and re-download
Right now if a file in the download cache fails the hash check, pip just gives up. It should instead purge that file from the cache and try again with a real download.

Any objections?



Discussion:
+1

+1 but make sure this is somewhere mentioned along the way, whether loud for users or just the log file, I don't care.

Ok, I'm starting work on a PR for this.

@jezdez I agree, I think since even usage of a file from the download cache is loud by default, this should definitely be as well.


Resolving patch:
diff --git a/pip/download.py b/pip/download.py
--- a/pip/download.py
+++ b/pip/download.py
@@ -13,10 +13,10 @@
 
 import pip
 
-from pip.backwardcompat import (xmlrpclib, urllib, urllib2, httplib,
+from pip.backwardcompat import (urllib, urllib2, httplib,
                                 urlparse, string_types, get_http_message_param,
                                 match_hostname, CertificateError)
-from pip.exceptions import InstallationError, PipError
+from pip.exceptions import InstallationError, HashMismatch
 from pip.util import (splitext, rmtree, format_size, display_path,
                       backup_dir, ask_path_exists, unpack_file,
                       create_download_cache_folder, cache_download)
@@ -441,11 +441,11 @@ def _check_hash(download_hash, link):
     if download_hash.digest_size != hashlib.new(link.hash_name).digest_size:
         logger.fatal("Hash digest size of the package %d (%s) doesn't match the expected hash name %s!"
                     % (download_hash.digest_size, link, link.hash_name))
-        raise InstallationError('Hash name mismatch for package %s' % link)
+        raise HashMismatch('Hash name mismatch for package %s' % link)
     if download_hash.hexdigest() != link.hash:
         logger.fatal("Hash of the package %s (%s) doesn't match the expected hash %s!"
                      % (link, download_hash, link.hash))
-        raise InstallationError('Bad %s hash for package %s' % (link.hash_name, link))
+        raise HashMismatch('Bad %s hash for package %s' % (link.hash_name, link))
 
 
 def _get_hash_from_file(target_file, link):
@@ -536,12 +536,21 @@ def _copy_file(filename, location, content_type, link):
 
 def unpack_http_url(link, location, download_cache, download_dir=None):
     temp_dir = tempfile.mkdtemp('-unpack', 'pip-')
+    temp_location = None
     target_url = link.url.split('#', 1)[0]
-    target_file = None
+
+    already_cached = False
+    cache_file = None
+    cache_content_type_file = None
     download_hash = None
     if download_cache:
-        target_file = os.path.join(download_cache,
+        cache_file = os.path.join(download_cache,
                                    urllib.quote(target_url, ''))
+        cache_content_type_file = cache_file + '.content-type'
+        already_cached = (
+            os.path.exists(cache_file) and
+            os.path.exists(cache_content_type_file)
+            )
         if not os.path.isdir(download_cache):
             create_download_cache_folder(download_cache)
 
@@ -551,23 +560,45 @@ def unpack_http_url(link, location, download_cache, download_dir=None):
         if not os.path.exists(already_downloaded):
             already_downloaded = None
 
-    if (target_file
-        and os.path.exists(target_file)
-        and os.path.exists(target_file + '.content-type')):
-        fp = open(target_file+'.content-type')
-        content_type = fp.read().strip()
-        fp.close()
-        if link.hash and link.hash_name:
-            download_hash = _get_hash_from_file(target_file, link)
-        temp_location = target_file
-        logger.notify('Using download cache from %s' % target_file)
-    elif already_downloaded:
+    if already_downloaded:
         temp_location = already_downloaded
-        content_type = mimetypes.guess_type(already_downloaded)
+        content_type = mimetypes.guess_type(already_downloaded)[0]
+        logger.notify('File was already downloaded %s' % already_downloaded)
         if link.hash:
             download_hash = _get_hash_from_file(temp_location, link)
-        logger.notify('File was already downloaded %s' % already_downloaded)
-    else:
+            try:
+                _check_hash(download_hash, link)
+            except HashMismatch:
+                logger.warn(
+                    'Previously-downloaded file %s has bad hash, '
+                    're-downloading.' % temp_location
+                    )
+                temp_location = None
+                os.unlink(already_downloaded)
+                already_downloaded = None
+
+    # We have a cached file, and we haven't already found a good downloaded copy
+    if already_cached and not temp_location:
+        with open(cache_content_type_file) as fp:
+            content_type = fp.read().strip()
+        temp_location = cache_file
+        logger.notify('Using download cache from %s' % cache_file)
+        if link.hash and link.hash_name:
+            download_hash = _get_hash_from_file(cache_file, link)
+            try:
+                _check_hash(download_hash, link)
+            except HashMismatch:
+                logger.warn(
+                    'Cached file %s has bad hash, '
+                    're-downloading.' % temp_location
+                    )
+                temp_location = None
+                os.unlink(cache_file)
+                os.unlink(cache_content_type_file)
+                already_cached = False
+
+    # We don't have either a cached or a downloaded copy
+    if not temp_location:
         resp = _get_response_from_url(target_url, link)
         content_type = resp.info().get('content-type', '')
         filename = link.filename  # fallback
@@ -589,14 +620,15 @@ def unpack_http_url(link, location, download_cache, download_dir=None):
                 filename += ext
         temp_location = os.path.join(temp_dir, filename)
         download_hash = _download_url(resp, link, temp_location)
-    if link.hash and link.hash_name:
-        _check_hash(download_hash, link)
+        if link.hash and link.hash_name:
+            _check_hash(download_hash, link)
+
     if download_dir and not already_downloaded:
         _copy_file(temp_location, download_dir, content_type, link)
     unpack_file(temp_location, location, content_type, link)
-    if target_file and target_file != temp_location:
-        cache_download(target_file, temp_location, content_type)
-    if target_file is None and not already_downloaded:
+    if cache_file and not already_cached:
+        cache_download(cache_file, temp_location, content_type)
+    if not (already_cached or already_downloaded):
         os.unlink(temp_location)
     os.rmdir(temp_dir)
 
diff --git a/pip/exceptions.py b/pip/exceptions.py
--- a/pip/exceptions.py
+++ b/pip/exceptions.py
@@ -1,6 +1,5 @@
 """Exceptions used throughout package"""
 
-import textwrap
 
 class PipError(Exception):
     """Base pip exception"""
@@ -34,3 +33,6 @@ class CommandError(PipError):
 class PreviousBuildDirError(PipError):
     """Raised when there's a previous conflicting build directory"""
 
+
+class HashMismatch(InstallationError):
+    """Distribution file hash values don't match."""
diff --git a/pip/util.py b/pip/util.py
--- a/pip/util.py
+++ b/pip/util.py
@@ -582,7 +582,6 @@ def cache_download(target_file, temp_location, content_type):
     fp = open(target_file+'.content-type', 'w')
     fp.write(content_type)
     fp.close()
-    os.unlink(temp_location)
 
 
 def unpack_file(filename, location, content_type, link):


Test patch:
diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py
--- a/tests/unit/test_download.py
+++ b/tests/unit/test_download.py
@@ -1,9 +1,11 @@
+import hashlib
 import os
 from shutil import rmtree
 from tempfile import mkdtemp
 
-import pip
 from mock import patch
+import pip
+from pip.backwardcompat import urllib, BytesIO, b
 from pip.download import (_get_response_from_url as _get_response_from_url_original,
                           path_to_url2, unpack_http_url, URLOpener)
 from pip.index import Link
@@ -34,3 +36,80 @@ def test_user_agent():
     opener = URLOpener().get_opener()
     user_agent = [x for x in opener.addheaders if x[0].lower() == "user-agent"][0]
     assert user_agent[1].startswith("pip/%s" % pip.__version__)
+
+
+def _write_file(fn, contents):
+    with open(fn, 'w') as fh:
+        fh.write(contents)
+
+
+class MockResponse(object):
+    def __init__(self, contents):
+        self._io = BytesIO(contents)
+
+    def read(self, *a, **kw):
+        return self._io.read(*a, **kw)
+
+
+@patch('pip.download.unpack_file')
+@patch('pip.download._get_response_from_url')
+def test_unpack_http_url_bad_cache_checksum(mock_get_response, mock_unpack_file):
+    """
+    If cached download has bad checksum, re-download.
+    """
+    base_url = 'http://www.example.com/somepackage.tgz'
+    contents = b('downloaded')
+    download_hash = hashlib.new('sha1', contents)
+    link = Link(base_url + '#sha1=' + download_hash.hexdigest())
+    response = mock_get_response.return_value = MockResponse(contents)
+    response.info = lambda: {'content-type': 'application/x-tar'}
+    response.geturl = lambda: base_url
+
+    cache_dir = mkdtemp()
+    try:
+        cache_file = os.path.join(cache_dir, urllib.quote(base_url, ''))
+        cache_ct_file = cache_file + '.content-type'
+        _write_file(cache_file, 'some contents')
+        _write_file(cache_ct_file, 'application/x-tar')
+
+        unpack_http_url(link, 'location', download_cache=cache_dir)
+
+        # despite existence of cached file with bad hash, downloaded again
+        mock_get_response.assert_called_once_with(base_url, link)
+        # cached file is replaced with newly downloaded file
+        with open(cache_file) as fh:
+            assert fh.read() == 'downloaded'
+
+    finally:
+        rmtree(cache_dir)
+
+
+@patch('pip.download.unpack_file')
+@patch('pip.download._get_response_from_url')
+def test_unpack_http_url_bad_downloaded_checksum(mock_get_response, mock_unpack_file):
+    """
+    If already-downloaded file has bad checksum, re-download.
+    """
+    base_url = 'http://www.example.com/somepackage.tgz'
+    contents = b('downloaded')
+    download_hash = hashlib.new('sha1', contents)
+    link = Link(base_url + '#sha1=' + download_hash.hexdigest())
+    response = mock_get_response.return_value = MockResponse(contents)
+    response.info = lambda: {'content-type': 'application/x-tar'}
+    response.geturl = lambda: base_url
+
+    download_dir = mkdtemp()
+    try:
+        downloaded_file = os.path.join(download_dir, 'somepackage.tgz')
+        _write_file(downloaded_file, 'some contents')
+
+        unpack_http_url(link, 'location', download_cache=None, download_dir=download_dir)
+
+        # despite existence of downloaded file with bad hash, downloaded again
+        mock_get_response.assert_called_once_with(base_url, link)
+        # cached file is replaced with newly downloaded file
+        with open(downloaded_file) as fh:
+            assert fh.read() == 'downloaded'
+
+    finally:
+        rmtree(download_dir)