Repository: getmoto/moto

Issue:
supporting utf-8 in key deletions
Fixes #1198
mock_s3 breaks delete_objects with Unicode
## Importing mock_s3 causes Unicode errors with delete_objects

Steps to reproduce (Python 2.7):
```
pip install boto3    # (1.4.7)
pip install moto     # (1.1.14)
python
>>> import boto3
>>> from moto import mock_s3
>>> client = boto3.client('s3')
>>> client.delete_objects(Bucket='bucket-name', Delete={'Objects': [{'Key': u'ó'}]})
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File ".../local/lib/python2.7/site-packages/botocore/client.py", line 312, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File ".../local/lib/python2.7/site-packages/botocore/client.py", line 588, in _make_api_call
    operation_model, request_dict)
  File ".../local/lib/python2.7/site-packages/botocore/endpoint.py", line 141, in make_request
    return self._send_request(request_dict, operation_model)
  File ".../local/lib/python2.7/site-packages/botocore/endpoint.py", line 170, in _send_request
    success_response, exception):
  File ".../local/lib/python2.7/site-packages/botocore/endpoint.py", line 249, in _needs_retry
    caught_exception=caught_exception, request_dict=request_dict)
  File ".../local/lib/python2.7/site-packages/botocore/hooks.py", line 227, in emit
    return self._emit(event_name, kwargs)
  File ".../local/lib/python2.7/site-packages/botocore/hooks.py", line 210, in _emit
    response = handler(**kwargs)
  File ".../local/lib/python2.7/site-packages/botocore/retryhandler.py", line 183, in __call__
    if self._checker(attempts, response, caught_exception):
  File ".../local/lib/python2.7/site-packages/botocore/retryhandler.py", line 251, in __call__
    caught_exception)
  File ".../local/lib/python2.7/site-packages/botocore/retryhandler.py", line 269, in _should_retry
    return self._checker(attempt_number, response, caught_exception)
  File ".../local/lib/python2.7/site-packages/botocore/retryhandler.py", line 317, in __call__
    caught_exception)
  File ".../local/lib/python2.7/site-packages/botocore/retryhandler.py", line 223, in __call__
    attempt_number, caught_exception)
  File ".../local/lib/python2.7/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception
    raise caught_exception
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 69: ordinal not in range(128)
```

https://github.com/boto/botocore/issues/1281


Discussion:

Resolving patch:
diff --git a/moto/core/models.py b/moto/core/models.py
--- a/moto/core/models.py
+++ b/moto/core/models.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 from __future__ import absolute_import
 
@@ -176,16 +177,49 @@ def fake_boto3_resource(*args, **kwargs):
             if 'endpoint_url' not in kwargs:
                 kwargs['endpoint_url'] = "http://localhost:5000"
             return real_boto3_resource(*args, **kwargs)
+
+        def fake_httplib_send_output(self, message_body=None, *args, **kwargs):
+            def _convert_to_bytes(mixed_buffer):
+                bytes_buffer = []
+                for chunk in mixed_buffer:
+                    if isinstance(chunk, six.text_type):
+                        bytes_buffer.append(chunk.encode('utf-8'))
+                    else:
+                        bytes_buffer.append(chunk)
+                msg = b"\r\n".join(bytes_buffer)
+                return msg
+
+            self._buffer.extend((b"", b""))
+            msg = _convert_to_bytes(self._buffer)
+            del self._buffer[:]
+            if isinstance(message_body, bytes):
+                msg += message_body
+                message_body = None
+            self.send(msg)
+            # if self._expect_header_set:
+            #     read, write, exc = select.select([self.sock], [], [self.sock], 1)
+            #     if read:
+            #         self._handle_expect_response(message_body)
+            #         return
+            if message_body is not None:
+                self.send(message_body)
+
         self._client_patcher = mock.patch('boto3.client', fake_boto3_client)
-        self._resource_patcher = mock.patch(
-            'boto3.resource', fake_boto3_resource)
+        self._resource_patcher = mock.patch('boto3.resource', fake_boto3_resource)
+        if six.PY2:
+            self._httplib_patcher = mock.patch('httplib.HTTPConnection._send_output', fake_httplib_send_output)
+
         self._client_patcher.start()
         self._resource_patcher.start()
+        if six.PY2:
+            self._httplib_patcher.start()
 
     def disable_patching(self):
         if self._client_patcher:
             self._client_patcher.stop()
             self._resource_patcher.stop()
+            if six.PY2:
+                self._httplib_patcher.stop()
 
 
 class Model(type):
diff --git a/moto/s3/responses.py b/moto/s3/responses.py
--- a/moto/s3/responses.py
+++ b/moto/s3/responses.py
@@ -8,6 +8,7 @@
 
 import xmltodict
 
+from moto.packages.httpretty.core import HTTPrettyRequest
 from moto.core.responses import _TemplateEnvironmentMixin
 
 from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys
@@ -113,7 +114,10 @@ def bucket_response(self, request, full_url, headers):
             return 200, {}, response.encode("utf-8")
         else:
             status_code, headers, response_content = response
-            return status_code, headers, response_content.encode("utf-8")
+            if not isinstance(response_content, six.binary_type):
+                response_content = response_content.encode("utf-8")
+
+            return status_code, headers, response_content
 
     def _bucket_response(self, request, full_url, headers):
         parsed_url = urlparse(full_url)
@@ -139,6 +143,7 @@ def _bucket_response(self, request, full_url, headers):
             body = b''
         if isinstance(body, six.binary_type):
             body = body.decode('utf-8')
+        body = u'{0}'.format(body).encode('utf-8')
 
         if method == 'HEAD':
             return self._bucket_response_head(bucket_name, headers)
@@ -209,7 +214,7 @@ def _bucket_response_get(self, bucket_name, querystring, headers):
             if not website_configuration:
                 template = self.response_template(S3_NO_BUCKET_WEBSITE_CONFIG)
                 return 404, {}, template.render(bucket_name=bucket_name)
-            return website_configuration
+            return 200, {}, website_configuration
         elif 'acl' in querystring:
             bucket = self.backend.get_bucket(bucket_name)
             template = self.response_template(S3_OBJECT_ACL_RESPONSE)
@@ -355,7 +360,7 @@ def _bucket_response_put(self, request, body, region_name, bucket_name, querystr
         if not request.headers.get('Content-Length'):
             return 411, {}, "Content-Length required"
         if 'versioning' in querystring:
-            ver = re.search('<Status>([A-Za-z]+)</Status>', body)
+            ver = re.search('<Status>([A-Za-z]+)</Status>', body.decode())
             if ver:
                 self.backend.set_bucket_versioning(bucket_name, ver.group(1))
                 template = self.response_template(S3_BUCKET_VERSIONING)
@@ -444,7 +449,12 @@ def _bucket_response_delete(self, body, bucket_name, querystring, headers):
     def _bucket_response_post(self, request, body, bucket_name, headers):
         if not request.headers.get('Content-Length'):
             return 411, {}, "Content-Length required"
-        path = request.path if hasattr(request, 'path') else request.path_url
+
+        if isinstance(request, HTTPrettyRequest):
+            path = request.path
+        else:
+            path = request.full_path if hasattr(request, 'full_path') else request.path_url
+
         if self.is_delete_keys(request, path, bucket_name):
             return self._bucket_response_delete_keys(request, body, bucket_name, headers)
 
@@ -454,6 +464,8 @@ def _bucket_response_post(self, request, body, bucket_name, headers):
             form = request.form
         else:
             # HTTPretty, build new form object
+            body = body.decode()
+
             form = {}
             for kv in body.split('&'):
                 k, v = kv.split('=')


Test patch:
diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py
--- a/tests/test_s3/test_s3.py
+++ b/tests/test_s3/test_s3.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-
 from __future__ import unicode_literals
 
 import datetime
@@ -1865,7 +1864,7 @@ def test_boto3_list_object_versions():
 def test_boto3_delete_markers():
     s3 = boto3.client('s3', region_name='us-east-1')
     bucket_name = 'mybucket'
-    key = 'key-with-versions'
+    key = u'key-with-versions-and-unicode-ó'
     s3.create_bucket(Bucket=bucket_name)
     s3.put_bucket_versioning(
         Bucket=bucket_name,
@@ -1880,10 +1879,9 @@ def test_boto3_delete_markers():
             Key=key,
             Body=body
         )
-    s3.delete_object(
-        Bucket=bucket_name,
-        Key=key
-    )
+
+    s3.delete_objects(Bucket=bucket_name, Delete={'Objects': [{'Key': key}]})
+
     with assert_raises(ClientError) as e:
         s3.get_object(
             Bucket=bucket_name,
@@ -1905,12 +1903,18 @@ def test_boto3_delete_markers():
         Bucket=bucket_name
     )
     response['Versions'].should.have.length_of(2)
-    response['Versions'][-1]['IsLatest'].should.be.true
-    response['Versions'][0]['IsLatest'].should.be.false
-    [(key_metadata['Key'], key_metadata['VersionId'])
-     for key_metadata in response['Versions']].should.equal(
-        [('key-with-versions', '0'), ('key-with-versions', '1')]
-    )
+
+    # We've asserted there is only 2 records so one is newest, one is oldest
+    latest = list(filter(lambda item: item['IsLatest'], response['Versions']))[0]
+    oldest = list(filter(lambda item: not item['IsLatest'], response['Versions']))[0]
+
+    # Double check ordering of version ID's
+    latest['VersionId'].should.equal('1')
+    oldest['VersionId'].should.equal('0')
+
+    # Double check the name is still unicode
+    latest['Key'].should.equal('key-with-versions-and-unicode-ó')
+    oldest['Key'].should.equal('key-with-versions-and-unicode-ó')
 
 
 @mock_s3