Repository: boto/botocore

Issue:
Unable to serialize map types for rest-xml
I don't believe there's a pre-existing issue filed for botocore against this.

This has been reported in the AWS CLI repo here: https://github.com/aws/aws-cli/issues/356

The fix will have to happen in botocore.

From looking into this, it looks like the Map parameter is missing a build_parameter_rest.  It has a build_parameter_query/build_parameter_json method, but the build_parameter_rest is inherited from the base class.  This means that the final serialization of the `metadata` param is:

```
{'headers': {u'x-amz-meta-': {'mykey2': 'myvalue2', 'mykey': 'myvalue'}, ...,
 'uri_params': {u'Bucket': 'test', u'Key': 'copied.txt'},
 'payload': <botocore.payload.XMLPayload object at 0x1034a2ed0>}
```

And because `x-amz-meta-` is a dictionary, this will raise an error later on in the auth module which expects the header to be a string.  The expected serialization is:

```
{'headers': {'x-amz-meta-mykey': 'myvalue', 'x-amz-meta-mykey2': 'myvalue2'}}
```

From what I can tell, s3 is the only rest-xml service that uses map types, so we can put in a specific case for rest-xml:

```
$ fgrep "rest-xml" *.json
cloudfront.json:  "type": "rest-xml",
route53.json:  "type": "rest-xml",
s3.json:  "type": "rest-xml",

$ fgrep '"type": "map"' s3.json cloudfront.json route53.json
s3.json:            "type": "map",
s3.json:            "type": "map",
s3.json:            "type": "map",
s3.json:            "type": "map",
s3.json:            "type": "map",
```



Discussion:
I might be missing something here, but it looks like we also don't properly parse `HeadObject` calls that return metadata:

```
>>> op.call(endpoint, bucket='bucket', key='copied.txt')[1]
...
2014-04-21 13:24:36,781 - botocore.response - DEBUG - Response Headers:
content-length: 0
accept-ranges: bytes
server: AmazonS3
last-modified: Mon, 21 Apr 2014 20:22:37 GMT
etag: "d41d8cd98f00b204e9800998ecf8427e"
date: Mon, 21 Apr 2014 20:24:37 GMT
x-amz-meta-mykey1: value1      <-------- this is not being returned.
content-type: application/json
...
{u'LastModified': 'Mon, 21 Apr 2014 20:22:37 GMT', u'AcceptRanges': 'bytes', u'ETag': '"d41d8cd98f00b204e9800998ecf8427e"', u'ContentType': 'application/json', u'ContentLength': '0'}
```


Resolving patch:
diff --git a/botocore/parameters.py b/botocore/parameters.py
--- a/botocore/parameters.py
+++ b/botocore/parameters.py
@@ -444,6 +444,21 @@ def build_parameter_json(self, value, built_params, label=''):
         for key in value:
             new_value[key] = value[key]
 
+    def build_parameter_rest(self, style, value, built_params, label=''):
+        # There's a special case for rest-xml with header locations
+        # that we we need to handle for maps.  If this is not the
+        # case we can defer to the base class's implementation of
+        # map parameters serialization.
+        if style == 'rest-xml' and getattr(self, 'location', '') == 'header':
+            prefix = getattr(self, 'location_name', '') or self.name
+            user_params = value
+            for key, value in user_params.items():
+                full_key_name = prefix + key
+                built_params['headers'][full_key_name] = value
+        else:
+            return super(MapParameter, self).build_parameter_rest(
+                style, value, build_params, label)
+
 
 class StructParameter(Parameter):
 
diff --git a/botocore/response.py b/botocore/response.py
--- a/botocore/response.py
+++ b/botocore/response.py
@@ -328,9 +328,23 @@ def merge_header_values(self, headers):
                 location = member.get('location')
                 if location == 'header':
                     location_name = member.get('location_name')
-                    if location_name in headers:
+                    if member['type'] == 'map':
+                        self._merge_map_header_values(headers, location_name,
+                                                      member_name, member)
+                    elif location_name in headers:
                         self.value[member_name] = headers[location_name]
 
+    def _merge_map_header_values(self, headers, location_name,
+                                 member_name, member):
+        final_map_value = {}
+        for header_name in headers:
+            if header_name.startswith(location_name):
+                header_value = headers[header_name]
+                actual_name = header_name[len(location_name):]
+                final_map_value[actual_name] = header_value
+        if final_map_value:
+            self.value[member_name] = final_map_value
+
 
 class JSONResponse(Response):
 


Test patch:
diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py
--- a/tests/integration/test_s3.py
+++ b/tests/integration/test_s3.py
@@ -396,6 +396,19 @@ def test_copy_with_quoted_char(self):
         data = response[1]
         self.assertEqual(data['Body'].read().decode('utf-8'), 'foo')
 
+    def test_copy_with_s3_metadata(self):
+        key_name = 'foo.txt'
+        self.create_object(key_name=key_name)
+        copied_key = 'copied.txt'
+        operation = self.service.get_operation('CopyObject')
+        http, parsed = operation.call(
+            self.endpoint, bucket=self.bucket_name, key=copied_key,
+            copy_source='%s/%s' % (self.bucket_name, key_name),
+            metadata_directive='REPLACE',
+            metadata={"mykey": "myvalue", "mykey2": "myvalue2"})
+        self.keys.append(copied_key)
+        self.assertEqual(http.status_code, 200)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/unit/response_parsing/test_response_parsing.py b/tests/unit/response_parsing/test_response_parsing.py
--- a/tests/unit/response_parsing/test_response_parsing.py
+++ b/tests/unit/response_parsing/test_response_parsing.py
@@ -175,7 +175,9 @@ def test_head_object(self):
              'Content-Type': 'binary/octet-stream',
              'accept-ranges': 'bytes',
              'Last-Modified': 'Tue, 20 Aug 2013 18:33:25 GMT',
-             'x-amz-server-side-encryption': 'AES256'
+             'x-amz-server-side-encryption': 'AES256',
+             'x-amz-meta-mykey1': 'value1',
+             'x-amz-meta-mykey2': 'value2',
              })
         http_response.content = ''
         http_response.request.method = 'HEAD'
@@ -185,8 +187,11 @@ def test_head_object(self):
                     "LastModified": "Tue, 20 Aug 2013 18:33:25 GMT",
                     "ContentLength": "265",
                     "ETag": '"40d06eb6194712ac1c915783004ef730"',
-                    "ServerSideEncryption": "AES256"
-                    }
+                    "ServerSideEncryption": "AES256",
+                    "Metadata": {
+                        'mykey1': 'value1',
+                        'mykey2': 'value2',
+                    }}
         response_data = get_response(self.session, put_object,
                                      http_response)[1]
         self.assertEqual(response_data, expected)
diff --git a/tests/unit/test_s3_operations.py b/tests/unit/test_s3_operations.py
--- a/tests/unit/test_s3_operations.py
+++ b/tests/unit/test_s3_operations.py
@@ -277,6 +277,23 @@ def test_put_object(self):
         self.assertEqual(params['uri_params'], uri_params)
         self.assertEqual(params['payload'].getvalue(), fp)
 
+    def test_copy_object(self):
+        operation = self.s3.get_operation('CopyObject')
+        bucket_name = 'mybucket'
+        key_name = 'key1'
+        copied_key= 'copied'
+        params = operation.build_parameters(
+            bucket=self.bucket_name, key=copied_key,
+            copy_source='%s/%s' % (bucket_name, key_name),
+            metadata_directive='REPLACE',
+            metadata={"mykey": "myvalue", "mykey2": "myvalue2"})
+        self.assertEqual(params['headers'], {
+            'x-amz-copy-source': 'mybucket/key1',
+            'x-amz-metadata-directive': 'REPLACE',
+            'x-amz-meta-mykey': 'myvalue',
+            'x-amz-meta-mykey2': 'myvalue2',
+        })
+
     def test_complete_multipart_upload(self):
         op = self.s3.get_operation('CompleteMultipartUpload')
         parts = {