Repository: boto/botocore
Pull request #807 (2016-02-24 20:56:24)

Verify MD5 is available
On some systems, such as a FIPS enabled system, MD5 hashing is unavailable. This commit will detect when that is the case, avoiding it where possible and raising a better error otherwise.

cc @jamesls @rayluo @kyleknap 


Patch:
diff --git a/botocore/compat.py b/botocore/compat.py
index 936589c63..028d02d65 100644
--- a/botocore/compat.py
+++ b/botocore/compat.py
@@ -16,10 +16,15 @@ import datetime
 import sys
 import inspect
 import warnings
+import hashlib
+import logging
 
 from botocore.vendored import six
+from botocore.exceptions import MD5UnavailableError
 from botocore.vendored.requests.packages.urllib3 import exceptions
 
+logger = logging.getLogger(__name__)
+
 
 if six.PY3:
     from six.moves import http_client
@@ -237,3 +242,30 @@ def total_seconds(delta):
     day_in_seconds = delta.days * 24 * 3600.0
     micro_in_seconds = delta.microseconds / 10.0**6
     return day_in_seconds + delta.seconds + micro_in_seconds
+
+
+# Checks to see if md5 is available on this system. A given system might not
+# have access to it for various reasons, such as FIPS mode being enabled.
+try:
+    hashlib.md5()
+    MD5_AVAILABLE = True
+except ValueError:
+    MD5_AVAILABLE = False
+
+
+def get_md5(*args, **kwargs):
+    """
+    Attempts to get an md5 hashing object.
+
+    :param raise_error_if_unavailable: raise an error if md5 is unavailable on
+        this system. If False, None will be returned if it is unavailable.
+    :type raise_error_if_unavailable: bool
+    :param args: Args to pass to the MD5 constructor
+    :param kwargs: Key word arguments to pass to the MD5 constructor
+    :return: An MD5 hashing object if available. If it is unavailable, None
+        is returned if raise_error_if_unavailable is set to False.
+    """
+    if MD5_AVAILABLE:
+        return hashlib.md5(*args, **kwargs)
+    else:
+        raise MD5UnavailableError()
diff --git a/botocore/exceptions.py b/botocore/exceptions.py
index 7ccc89265..c4f1404c9 100644
--- a/botocore/exceptions.py
+++ b/botocore/exceptions.py
@@ -367,3 +367,7 @@ class InvalidConfigError(BotoCoreError):
 
 class RefreshWithMFAUnsupportedError(BotoCoreError):
     fmt = 'Cannot refresh credentials: MFA token required.'
+
+
+class MD5UnavailableError(BotoCoreError):
+    fmt = "This system does not support MD5 generation."
diff --git a/botocore/handlers.py b/botocore/handlers.py
index 1b8f07b9a..7a384e64f 100644
--- a/botocore/handlers.py
+++ b/botocore/handlers.py
@@ -17,7 +17,6 @@ This module contains builtin handlers for events emitted by botocore.
 """
 
 import base64
-import hashlib
 import logging
 import xml.etree.cElementTree
 import copy
@@ -25,7 +24,7 @@ import re
 import warnings
 
 from botocore.compat import urlsplit, urlunsplit, unquote, \
-    json, quote, six, unquote_str, ensure_bytes
+    json, quote, six, unquote_str, ensure_bytes, get_md5, MD5_AVAILABLE
 from botocore.docs.utils import AutoPopulatedParam
 from botocore.docs.utils import HideParamFromOperations
 from botocore.docs.utils import AppendParamDocumentation
@@ -134,13 +133,13 @@ def calculate_md5(params, **kwargs):
 
 
 def _calculate_md5_from_bytes(body_bytes):
-    md5 = hashlib.md5(body_bytes)
+    md5 = get_md5(body_bytes)
     return md5.digest()
 
 
 def _calculate_md5_from_file(fileobj):
     start_position = fileobj.tell()
-    md5 = hashlib.md5()
+    md5 = get_md5()
     for chunk in iter(lambda: fileobj.read(1024 * 1024), b''):
         md5.update(chunk)
     fileobj.seek(start_position)
@@ -150,7 +149,7 @@ def _calculate_md5_from_file(fileobj):
 def conditionally_calculate_md5(params, **kwargs):
     """Only add a Content-MD5 when not using sigv4"""
     signer = kwargs['request_signer']
-    if signer.signature_version not in ['v4', 's3v4']:
+    if signer.signature_version not in ['v4', 's3v4'] and MD5_AVAILABLE:
         calculate_md5(params, **kwargs)
 
 
@@ -188,13 +187,14 @@ def copy_source_sse_md5(params, **kwargs):
 def _sse_md5(params, sse_member_prefix='SSECustomer'):
     if not _needs_s3_sse_customization(params, sse_member_prefix):
         return
+
     sse_key_member = sse_member_prefix + 'Key'
     sse_md5_member = sse_member_prefix + 'KeyMD5'
     key_as_bytes = params[sse_key_member]
     if isinstance(key_as_bytes, six.text_type):
         key_as_bytes = key_as_bytes.encode('utf-8')
     key_md5_str = base64.b64encode(
-        hashlib.md5(key_as_bytes).digest()).decode('utf-8')
+        get_md5(key_as_bytes).digest()).decode('utf-8')
     key_b64_encoded = base64.b64encode(key_as_bytes).decode('utf-8')
     params[sse_key_member] = key_b64_encoded
     params[sse_md5_member] = key_md5_str


Test patch:
diff --git a/tests/unit/test_compat.py b/tests/unit/test_compat.py
index fbaae3369..1683ed5b2 100644
--- a/tests/unit/test_compat.py
+++ b/tests/unit/test_compat.py
@@ -12,8 +12,12 @@
 # language governing permissions and limitations under the License.
 
 import datetime
+import mock
 
-from botocore.compat import total_seconds, unquote_str, six, ensure_bytes
+from botocore.exceptions import MD5UnavailableError
+from botocore.compat import (
+    total_seconds, unquote_str, six, ensure_bytes, get_md5
+)
 
 from tests import BaseEnvVar, unittest
 
@@ -78,3 +82,16 @@ class TestEnsureBytes(unittest.TestCase):
         value = 500
         with self.assertRaises(ValueError):
             ensure_bytes(value)
+
+
+class TestGetMD5(unittest.TestCase):
+    def test_available(self):
+        md5 = mock.Mock()
+        with mock.patch('botocore.compat.MD5_AVAILABLE', True):
+            with mock.patch('hashlib.md5', mock.Mock(return_value=md5)):
+                self.assertEqual(get_md5(), md5)
+
+    def test_unavailable_raises_error(self):
+        with mock.patch('botocore.compat.MD5_AVAILABLE', False):
+            with self.assertRaises(MD5UnavailableError):
+                get_md5()
diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py
index bcbbf7668..9d56765e2 100644
--- a/tests/unit/test_handlers.py
+++ b/tests/unit/test_handlers.py
@@ -20,7 +20,7 @@ import os
 
 import botocore
 import botocore.session
-from botocore.exceptions import ParamValidationError
+from botocore.exceptions import ParamValidationError, MD5UnavailableError
 from botocore.awsrequest import AWSRequest
 from botocore.compat import quote, six
 from botocore.model import OperationModel, ServiceModel
@@ -199,45 +199,6 @@ class TestHandlers(BaseSessionTest):
         # object is None.  We need to handle this case.
         handlers.check_for_200_error(None)
 
-    def test_sse_params(self):
-        for op in ('HeadObject', 'GetObject', 'PutObject', 'CopyObject',
-                   'CreateMultipartUpload', 'UploadPart', 'UploadPartCopy'):
-            event = 'before-parameter-build.s3.%s' % op
-            params = {'SSECustomerKey': b'bar',
-                      'SSECustomerAlgorithm': 'AES256'}
-            self.session.emit(event, params=params, model=mock.Mock())
-            self.assertEqual(params['SSECustomerKey'], 'YmFy')
-            self.assertEqual(params['SSECustomerKeyMD5'],
-                             'N7UdGUp1E+RbVvZSTy1R8g==')
-
-    def test_sse_params_as_str(self):
-        event = 'before-parameter-build.s3.PutObject'
-        params = {'SSECustomerKey': 'bar',
-                  'SSECustomerAlgorithm': 'AES256'}
-        self.session.emit(event, params=params, model=mock.Mock())
-        self.assertEqual(params['SSECustomerKey'], 'YmFy')
-        self.assertEqual(params['SSECustomerKeyMD5'],
-                         'N7UdGUp1E+RbVvZSTy1R8g==')
-
-    def test_copy_source_sse_params(self):
-        for op in ['CopyObject', 'UploadPartCopy']:
-            event = 'before-parameter-build.s3.%s' % op
-            params = {'CopySourceSSECustomerKey': b'bar',
-                      'CopySourceSSECustomerAlgorithm': 'AES256'}
-            self.session.emit(event, params=params, model=mock.Mock())
-            self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
-            self.assertEqual(params['CopySourceSSECustomerKeyMD5'],
-                             'N7UdGUp1E+RbVvZSTy1R8g==')
-
-    def test_copy_source_sse_params_as_str(self):
-        event = 'before-parameter-build.s3.CopyObject'
-        params = {'CopySourceSSECustomerKey': 'bar',
-                  'CopySourceSSECustomerAlgorithm': 'AES256'}
-        self.session.emit(event, params=params, model=mock.Mock())
-        self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
-        self.assertEqual(params['CopySourceSSECustomerKeyMD5'],
-                         'N7UdGUp1E+RbVvZSTy1R8g==')
-
     def test_route53_resource_id(self):
         event = 'before-parameter-build.route53.GetHostedZone'
         params = {'Id': '/hostedzone/ABC123',
@@ -509,60 +470,7 @@ class TestHandlers(BaseSessionTest):
         handlers.switch_host_with_param(request, 'PredictEndpoint')
         self.assertEqual(request.url, new_endpoint)
 
-    def test_does_not_add_md5_when_v4(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-        handlers.conditionally_calculate_md5(request_dict,
-                                             request_signer=request_signer)
-        self.assertTrue('Content-MD5' not in request_dict['headers'])
 
-    def test_does_not_add_md5_when_s3v4(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-        handlers.conditionally_calculate_md5(request_dict,
-                                             request_signer=request_signer)
-        self.assertTrue('Content-MD5' not in request_dict['headers'])
-
-    def test_adds_md5_when_not_v4(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-        handlers.conditionally_calculate_md5(request_dict,
-                                             request_signer=request_signer)
-        self.assertTrue('Content-MD5' in request_dict['headers'])
-
-    def test_adds_md5_with_file_like_body(self):
-        request_dict = {
-            'body': six.BytesIO(b'foobar'),
-            'headers': {}
-        }
-        handlers.calculate_md5(request_dict)
-        self.assertEqual(request_dict['headers']['Content-MD5'],
-                         'OFj2IjCsPJFfMAxmQxLGPw==')
-
-    def test_adds_md5_with_bytes_object(self):
-        request_dict = {
-            'body': b'foobar',
-            'headers': {}
-        }
-        handlers.calculate_md5(request_dict)
-        self.assertEqual(
-            request_dict['headers']['Content-MD5'],
-            'OFj2IjCsPJFfMAxmQxLGPw==')
 
     def test_invalid_char_in_bucket_raises_exception(self):
         params = {
@@ -735,3 +643,164 @@ class TestRetryHandlerOrder(BaseSessionTest):
         self.assertTrue(s3_200_handler < general_retry_handler,
                         "S3 200 error handler was supposed to be before "
                         "the general retry handler, but it was not.")
+
+
+class BaseMD5Test(BaseSessionTest):
+    def setUp(self, **environ):
+        super(BaseMD5Test, self).setUp(**environ)
+        self.md5_object = mock.Mock()
+        self.md5_digest = mock.Mock(return_value=b'foo')
+        self.md5_object.digest = self.md5_digest
+        md5_builder = mock.Mock(return_value=self.md5_object)
+        self.md5_patch = mock.patch('hashlib.md5', md5_builder)
+        self.md5_patch.start()
+        self._md5_available_patch = None
+        self.set_md5_available()
+
+    def tearDown(self):
+        super(BaseMD5Test, self).tearDown()
+        self.md5_patch.stop()
+        if self._md5_available_patch:
+            self._md5_available_patch.stop()
+
+    def set_md5_available(self, is_available=True):
+        if self._md5_available_patch:
+            self._md5_available_patch.stop()
+
+        self._md5_available_patch = \
+            mock.patch('botocore.compat.MD5_AVAILABLE', is_available)
+        self._md5_available_patch.start()
+
+
+class TestSSEMD5(BaseMD5Test):
+    def test_raises_error_when_md5_unavailable(self):
+        self.set_md5_available(False)
+
+        with self.assertRaises(MD5UnavailableError):
+            handlers.sse_md5({'SSECustomerKey': b'foo'})
+
+        with self.assertRaises(MD5UnavailableError):
+            handlers.copy_source_sse_md5({'CopySourceSSECustomerKey': b'foo'})
+
+    def test_sse_params(self):
+        for op in ('HeadObject', 'GetObject', 'PutObject', 'CopyObject',
+                   'CreateMultipartUpload', 'UploadPart', 'UploadPartCopy'):
+            event = 'before-parameter-build.s3.%s' % op
+            params = {'SSECustomerKey': b'bar',
+                      'SSECustomerAlgorithm': 'AES256'}
+            self.session.emit(event, params=params, model=mock.Mock())
+            self.assertEqual(params['SSECustomerKey'], 'YmFy')
+            self.assertEqual(params['SSECustomerKeyMD5'], 'Zm9v')
+
+    def test_sse_params_as_str(self):
+        event = 'before-parameter-build.s3.PutObject'
+        params = {'SSECustomerKey': 'bar',
+                  'SSECustomerAlgorithm': 'AES256'}
+        self.session.emit(event, params=params, model=mock.Mock())
+        self.assertEqual(params['SSECustomerKey'], 'YmFy')
+        self.assertEqual(params['SSECustomerKeyMD5'], 'Zm9v')
+
+    def test_copy_source_sse_params(self):
+        for op in ['CopyObject', 'UploadPartCopy']:
+            event = 'before-parameter-build.s3.%s' % op
+            params = {'CopySourceSSECustomerKey': b'bar',
+                      'CopySourceSSECustomerAlgorithm': 'AES256'}
+            self.session.emit(event, params=params, model=mock.Mock())
+            self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
+            self.assertEqual(params['CopySourceSSECustomerKeyMD5'], 'Zm9v')
+
+    def test_copy_source_sse_params_as_str(self):
+        event = 'before-parameter-build.s3.CopyObject'
+        params = {'CopySourceSSECustomerKey': 'bar',
+                  'CopySourceSSECustomerAlgorithm': 'AES256'}
+        self.session.emit(event, params=params, model=mock.Mock())
+        self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
+        self.assertEqual(params['CopySourceSSECustomerKeyMD5'], 'Zm9v')
+
+
+class TestAddMD5(BaseMD5Test):
+    def test_does_not_add_md5_when_v4(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+        handlers.conditionally_calculate_md5(request_dict,
+                                             request_signer=request_signer)
+        self.assertTrue('Content-MD5' not in request_dict['headers'])
+
+    def test_does_not_add_md5_when_s3v4(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+        handlers.conditionally_calculate_md5(request_dict,
+                                             request_signer=request_signer)
+        self.assertTrue('Content-MD5' not in request_dict['headers'])
+
+    def test_conditional_does_not_add_when_md5_unavailable(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+
+        self.set_md5_available(False)
+        with mock.patch('botocore.handlers.MD5_AVAILABLE', False):
+            handlers.conditionally_calculate_md5(
+                request_dict, request_signer=request_signer)
+            self.assertFalse('Content-MD5' in request_dict['headers'])
+
+    def test_add_md5_raises_error_when_md5_unavailable(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+
+        self.set_md5_available(False)
+        with self.assertRaises(MD5UnavailableError):
+            handlers.calculate_md5(
+                request_dict, request_signer=request_signer)
+
+    def test_adds_md5_when_not_v4(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+        handlers.conditionally_calculate_md5(request_dict,
+                                             request_signer=request_signer)
+        self.assertTrue('Content-MD5' in request_dict['headers'])
+
+    def test_add_md5_with_file_like_body(self):
+        request_dict = {
+            'body': six.BytesIO(b'foobar'),
+            'headers': {}
+        }
+        self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
+        handlers.calculate_md5(request_dict)
+        self.assertEqual(request_dict['headers']['Content-MD5'],
+                         'OFj2IjCsPJFfMAxmQxLGPw==')
+
+    def test_add_md5_with_bytes_object(self):
+        request_dict = {
+            'body': b'foobar',
+            'headers': {}
+        }
+        self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
+        handlers.calculate_md5(request_dict)
+        self.assertEqual(
+            request_dict['headers']['Content-MD5'],
+            'OFj2IjCsPJFfMAxmQxLGPw==')