Repository: boto/botocore
Pull request #765 (2016-01-21 01:22:36)

Adding missing url decoding from S3 responses.
The previous commit to add automatic endcoding and decoding of S3 responses missed a few keys: https://github.com/boto/botocore/pull/726

Specifically, we were not decoding Marker, Delimiter, NextMarker, and CommonPrefixes[*].Prefix. The decode_list_object handler has been updated to decode each of these members in the parsed response if they are present.

This change will fix a CLI issue: https://github.com/boto/botocore/issues/762

For more information, see http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html

(supersedes and builds on https://github.com/boto/botocore/pull/752)

@jamesls @kyleknap @JordonPhillips @rayluo 


Patch:
diff --git a/botocore/handlers.py b/botocore/handlers.py
index 41d333488..cf4d4233d 100644
--- a/botocore/handlers.py
+++ b/botocore/handlers.py
@@ -600,11 +600,24 @@ def decode_list_object(parsed, context, **kwargs):
     # This is needed because we are passing url as the encoding type. Since the
     # paginator is based on the key, we need to handle it before it can be
     # round tripped.
-    if 'Contents' in parsed and parsed.get('EncodingType') == 'url' and \
+    #
+    # From the documentation: If you specify encoding-type request parameter,
+    # Amazon S3 includes this element in the response, and returns encoded key
+    # name values in the following response elements:
+    # Delimiter, Marker, Prefix, NextMarker, Key.
+    if parsed.get('EncodingType') == 'url' and \
                     context.get('EncodingTypeAutoSet'):
-        for content in parsed['Contents']:
-            content['Key'] = unquote_str(content['Key'])
-
+        # URL decode top-level keys in the response if present.
+        top_level_keys = ['Delimiter', 'Marker', 'NextMarker']
+        for key in top_level_keys:
+            if key in parsed:
+                parsed[key] = unquote_str(parsed[key])
+        # URL decode nested keys from the response if present.
+        nested_keys = [('Contents', 'Key'), ('CommonPrefixes', 'Prefix')]
+        for (top_key, child_key) in nested_keys:
+            if top_key in parsed:
+                for member in parsed[top_key]:
+                    member[child_key] = unquote_str(member[child_key])
 
 # This is a list of (event_name, handler).
 # When a Session is created, everything in this list will be


Test patch:
diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py
index 1777c72aa..2c4305ca9 100644
--- a/tests/unit/test_handlers.py
+++ b/tests/unit/test_handlers.py
@@ -620,6 +620,43 @@ class TestHandlers(BaseSessionTest):
         handlers.decode_list_object(parsed, context={})
         self.assertEqual(parsed['Contents'][0]['Key'], u'%C3%A7%C3%B6s%25asd')
 
+    def test_decode_list_objects_with_marker(self):
+        parsed = {
+            'Marker': "%C3%A7%C3%B6s%25%20asd%08+c",
+            'EncodingType': 'url',
+        }
+        context = {'EncodingTypeAutoSet': True}
+        handlers.decode_list_object(parsed, context=context)
+        self.assertEqual(parsed['Marker'], u'\xe7\xf6s% asd\x08 c')
+
+    def test_decode_list_objects_with_nextmarker(self):
+        parsed = {
+            'NextMarker': "%C3%A7%C3%B6s%25%20asd%08+c",
+            'EncodingType': 'url',
+        }
+        context = {'EncodingTypeAutoSet': True}
+        handlers.decode_list_object(parsed, context=context)
+        self.assertEqual(parsed['NextMarker'], u'\xe7\xf6s% asd\x08 c')
+
+    def test_decode_list_objects_with_common_prefixes(self):
+        parsed = {
+            'CommonPrefixes': [{'Prefix': "%C3%A7%C3%B6s%25%20asd%08+c"}],
+            'EncodingType': 'url',
+        }
+        context = {'EncodingTypeAutoSet': True}
+        handlers.decode_list_object(parsed, context=context)
+        self.assertEqual(parsed['CommonPrefixes'][0]['Prefix'],
+                         u'\xe7\xf6s% asd\x08 c')
+
+    def test_decode_list_objects_with_delimiter(self):
+        parsed = {
+            'Delimiter': "%C3%A7%C3%B6s%25%20asd%08+c",
+            'EncodingType': 'url',
+        }
+        context = {'EncodingTypeAutoSet': True}
+        handlers.decode_list_object(parsed, context=context)
+        self.assertEqual(parsed['Delimiter'], u'\xe7\xf6s% asd\x08 c')
+
 
 class TestRetryHandlerOrder(BaseSessionTest):
     def get_handler_names(self, responses):