Repository: boto/boto

Issue:
boto.glacier.utils.compute_hashes_from_fileobj No Longer Works with Binary Files
Commit a4c9a781f47a61ddde2b3a3802b93c1ed29cdf16 added a `.encode('utf-8')` to the result of the first read in `boto.glacier.utils.compute_hashes_from_fileobj` (although it doesn't attempt to encode the results of the reads in the loop below that).

This breaks for binary files for me with python 2.6.6:

```
(glacier)[cperl@localhost ~]$ dd if=/dev/urandom of=/tmp/foo.bin bs=1M count=1
1+0 records in
1+0 records out
1048576 bytes (1.0 MB) copied, 0.110299 s, 9.5 MB/s

(glacier)[cperl@localhost ~]$ python
Python 2.6.6 (r266:84292, Nov 22 2013, 12:16:22) 
[GCC 4.4.7 20120313 (Red Hat 4.4.7-4)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import boto.glacier.utils
>>> with open("/tmp/foo.bin", 'r') as f:
...     boto.glacier.utils.compute_hashes_from_fileobj(f)
... 
Traceback (most recent call last):
  File "<stdin>", line 2, in <module>
  File "/home/cperl/virtualenv/glacier/lib/python2.6/site-packages/boto/glacier/utils.py", line 127, in compute_hashes_from_fileobj
    chunk = fileobj.read(chunk_size).encode('utf-8')
UnicodeDecodeError: 'ascii' codec can't decode byte 0xf4 in position 0: ordinal not in range(128)
```



Discussion:

Resolving patch:
diff --git a/boto/glacier/utils.py b/boto/glacier/utils.py
--- a/boto/glacier/utils.py
+++ b/boto/glacier/utils.py
@@ -23,6 +23,8 @@
 import math
 import binascii
 
+from boto.compat import six
+
 
 _MEGABYTE = 1024 * 1024
 DEFAULT_PART_SIZE = 4 * _MEGABYTE
@@ -122,10 +124,19 @@ def compute_hashes_from_fileobj(fileobj, chunk_size=1024 * 1024):
         are returned in hex.
 
     """
+    # Python 3+, not binary
+    if six.PY3 and hasattr(fileobj, 'mode') and 'b' not in fileobj.mode:
+        raise ValueError('File-like object must be opened in binary mode!')
+
     linear_hash = hashlib.sha256()
     chunks = []
-    chunk = fileobj.read(chunk_size).encode('utf-8')
+    chunk = fileobj.read(chunk_size)
     while chunk:
+        # It's possible to get a file-like object that has no mode (checked
+        # above) and returns something other than bytes (e.g. str). So here
+        # we try to catch that and encode to bytes.
+        if not isinstance(chunk, bytes):
+            chunk = chunk.encode(getattr(fileobj, 'encoding', '') or 'utf-8')
         linear_hash.update(chunk)
         chunks.append(hashlib.sha256(chunk).digest())
         chunk = fileobj.read(chunk_size)


Test patch:
diff --git a/tests/unit/glacier/test_utils.py b/tests/unit/glacier/test_utils.py
--- a/tests/unit/glacier/test_utils.py
+++ b/tests/unit/glacier/test_utils.py
@@ -19,13 +19,16 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 #
-import time
 import logging
+import os
+import tempfile
+import time
 from hashlib import sha256
 from tests.unit import unittest
 
+from boto.compat import BytesIO, six, StringIO
 from boto.glacier.utils import minimum_part_size, chunk_hashes, tree_hash, \
-        bytes_to_hex
+        bytes_to_hex, compute_hashes_from_fileobj
 
 
 class TestPartSizeCalculations(unittest.TestCase):
@@ -114,3 +117,49 @@ def test_empty_tree_hash(self):
         self.assertEqual(
             self.calculate_tree_hash(''),
             b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
+
+
+class TestFileHash(unittest.TestCase):
+    def _gen_data(self):
+        # Generate some pseudo-random bytes of data. We include the
+        # hard-coded blob as an example that fails to decode via UTF-8.
+        return os.urandom(5000) + b'\xc2\x00'
+
+    def test_compute_hash_tempfile(self):
+        # Compute a hash from a file object. On Python 2 this uses a non-
+        # binary mode. On Python 3, however, binary mode is required for
+        # binary files. If not used, you will get UTF-8 code errors.
+        if six.PY2:
+            mode = "w+"
+        else:
+            mode = "wb+"
+
+        with tempfile.TemporaryFile(mode=mode) as f:
+            f.write(self._gen_data())
+            f.seek(0)
+
+            compute_hashes_from_fileobj(f, chunk_size=512)
+
+    @unittest.skipUnless(six.PY3, 'Python 3 requires reading binary!')
+    def test_compute_hash_tempfile_py3(self):
+        # Note the missing 'b' in the mode!
+        with tempfile.TemporaryFile(mode='w+') as f:
+            with self.assertRaises(ValueError):
+                compute_hashes_from_fileobj(f, chunk_size=512)
+
+        # What about file-like objects without a mode? If it has an
+        # encoding we use it, otherwise attempt UTF-8 encoding to
+        # bytes for hashing.
+        f = StringIO('test data' * 500)
+        compute_hashes_from_fileobj(f, chunk_size=512)
+
+    @unittest.skipUnless(six.PY2, 'Python 3 requires reading binary!')
+    def test_compute_hash_stringio(self):
+        # Python 2 binary data in StringIO example
+        f = StringIO(self._gen_data())
+        compute_hashes_from_fileobj(f, chunk_size=512)
+
+    def test_compute_hash_bytesio(self):
+        # Compute a hash from a file-like BytesIO object.
+        f = BytesIO(self._gen_data())
+        compute_hashes_from_fileobj(f, chunk_size=512)
diff --git a/tests/unit/glacier/test_vault.py b/tests/unit/glacier/test_vault.py
--- a/tests/unit/glacier/test_vault.py
+++ b/tests/unit/glacier/test_vault.py
@@ -44,7 +44,9 @@ def setUp(self):
     def tearDown(self):
         self.size_patch.stop()
 
-    def test_upload_archive_small_file(self):
+    @mock.patch('boto.glacier.vault.compute_hashes_from_fileobj',
+                return_value=[b'abc', b'123'])
+    def test_upload_archive_small_file(self, compute_hashes):
         self.getsize.return_value = 1
 
         self.api.upload_archive.return_value = {'ArchiveId': 'archive_id'}