Repository: boto/botocore

Issue:
Errors not being parsed for streaming responses
If an error occurs, we don't parse the body of a streaming response.  For example if an `s3.GetObject` fails because of an auth issue, we don't get the parsed error response:

```
>>> import botocore.session
>>> session = botocore.session.get_session()
>>> s3 = session.get_service('s3')
>>> endpoint = s3.get_endpoint('us-east-1')
>>> operation = s3.get_operation('GetObject')
>>> response = operation.call(endpoint, bucket='testbucket', key='testkey')
>>> response[0].status_code
403
>>> response[1]
{u'Body': <botocore.vendored.requests.packages.urllib3.response.HTTPResponse object at 0x10c3747d0>, u'ContentType': 'application/xml'}
```

I can read the body to get the raw XML error:

```
>>> response[1]['Body'].read()
'<?xml version="1.0" encoding="UTF-8"?>\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><StringToSignBytes>sequence of bytes</StringToSignBytes><RequestId>requestid</RequestId><HostId>hostid</HostId><SignatureProvided>signature</SignatureProvided><StringToSign>string to sign</StringToSign><AWSAccessKeyId>access key</AWSAccessKeyId></Error>'
```

This also makes it difficult for code that uses botocore to present error messages in this scenario.  For example, in the CLI, the error you get for an auth error with `get-object` is:

```
 aws s3api get-object --bucket bucket  --key key /tmp/asdfasdfasdfasdfsdf
A client error (Unknown) occurred when calling the GetObject operation: Unknown
```



Discussion:

Resolving patch:
diff --git a/botocore/response.py b/botocore/response.py
--- a/botocore/response.py
+++ b/botocore/response.py
@@ -464,12 +464,29 @@ def get_response(session, operation, http_response):
         content_type = content_type.split(';')[0]
         logger.debug('Content type from response: %s', content_type)
     if operation.is_streaming():
+        logger.debug(
+            "Response Headers:\n%s",
+            '\n'.join("%s: %s" % (k, v) for k, v in http_response.headers.items()))
         streaming_response = StreamingResponse(session, operation)
         streaming_response.parse(
             http_response.headers,
             StreamingBody(http_response.raw,
                           http_response.headers.get('content-length')))
-        return (http_response, streaming_response.get_value())
+        if http_response.ok:
+            return (http_response, streaming_response.get_value())
+        else:
+            xml_response = XmlResponse(session, operation)
+            body = streaming_response.get_value()['Body'].read()
+            # For streaming response, response body might be binary data,
+            # so we log response body only when error happens.
+            logger.debug("Response Body:\n%s", body)
+            if body:
+                try:
+                    xml_response.parse(body, encoding)
+                except xml.etree.cElementTree.ParseError as err:
+                    raise xml.etree.cElementTree.ParseError(
+                        "Error parsing XML response: %s\nXML received:\n%s" % (err, body))
+            return (http_response, xml_response.get_value())
     body = http_response.content
     if not http_response.request.method == 'HEAD':
         _validate_content_length(


Test patch:
diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py
--- a/tests/unit/test_response.py
+++ b/tests/unit/test_response.py
@@ -13,10 +13,28 @@
 
 from tests import unittest
 
-import mock
 import six
+import botocore
 from botocore import response
 from botocore.exceptions import IncompleteReadError
+from botocore.vendored.requests.models import Response, Request
+
+XMLBODY1 = (b'<?xml version="1.0" encoding="UTF-8"?><Error>'
+            b'<Code>AccessDenied</Code>'
+            b'<Message>Access Denied</Message>'
+            b'<RequestId>XXXXXXXXXXXXXXXX</RequestId>'
+            b'<HostId>AAAAAAAAAAAAAAAAAAA</HostId>'
+            b'</Error>')
+
+XMLBODY2 = (b'<?xml version="1.0" encoding="UTF-8"?>'
+            b'<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
+            b'<Name>mybucket</Name><Prefix></Prefix><Marker></Marker>'
+            b'<MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>'
+            b'<Contents><Key>test.png</Key><LastModified>2014-03-01T17:06:40.000Z</LastModified>'
+            b'<ETag>&quot;00000000000000000000000000000000&quot;</ETag><Size>6702</Size>'
+            b'<Owner><ID>AAAAAAAAAAAAAAAAAAA</ID>'
+            b'<DisplayName>dummy</DisplayName></Owner>'
+            b'<StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>')
 
 
 class TestStreamWrapper(unittest.TestCase):
@@ -39,3 +57,122 @@ def test_streaming_body_with_single_read(self):
         stream = response.StreamingBody(body, content_length=10)
         with self.assertRaises(IncompleteReadError):
             stream.read()
+
+class TestGetResponse(unittest.TestCase):
+    def test_get_response_streaming_ok(self):
+        http_response = Response()
+        http_response.headers = {
+            'content-type': 'image/png',
+            'server': 'AmazonS3',
+            'AcceptRanges': 'bytes',
+            'transfer-encoding': 'chunked',
+            'ETag': '"00000000000000000000000000000000"',
+        }
+        http_response.raw = six.BytesIO(b'\x89PNG\r\n\x1a\n\x00\x00')
+
+        http_response.status_code = 200
+        http_response.reason = 'OK'
+
+        session = botocore.session.get_session()
+        s3 = session.get_service('s3')
+        operation = s3.get_operation('GetObject')
+
+        res = response.get_response(session, operation, http_response)
+        self.assertTrue(isinstance(res[1]['Body'], response.StreamingBody))
+        self.assertEqual(res[1]['ETag'],
+                         '"00000000000000000000000000000000"')
+
+    def test_get_response_streaming_ng(self):
+        http_response = Response()
+        http_response.headers = {
+            'content-type': 'application/xml',
+            'date': 'Sat, 08 Mar 2014 12:05:44 GMT',
+            'server': 'AmazonS3',
+            'transfer-encoding': 'chunked',
+            'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA',
+            'x-amz-request-id': 'XXXXXXXXXXXXXXXX'}
+        http_response.raw = six.BytesIO(XMLBODY1)
+        http_response.status_code = 403
+        http_response.reason = 'Forbidden'
+
+        session = botocore.session.get_session()
+        s3 = session.get_service('s3')
+        operation = s3.get_operation('GetObject') # streaming operation
+
+        self.assertEqual(
+            response.get_response(session, operation, http_response)[1], 
+            {u'Body': None,
+              'Errors': [{'HostId': 'AAAAAAAAAAAAAAAAAAA',
+                          'Message': 'Access Denied',
+                          'Code': 'AccessDenied',
+                          'RequestId': 'XXXXXXXXXXXXXXXX'}],
+              'ResponseMetadata': {},
+             u'Metadata': {}}
+            )
+
+    def test_get_response_nonstreaming_ok(self):
+        http_response = Response()
+        http_response.headers = {
+            'content-type': 'application/xml',
+            'date': 'Sun, 09 Mar 2014 02:55:43 GMT',
+            'server': 'AmazonS3',
+            'transfer-encoding': 'chunked',
+            'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA',
+            'x-amz-request-id': 'XXXXXXXXXXXXXXXX'}
+        http_response.raw = six.BytesIO(XMLBODY1)
+        http_response.status_code = 403
+        http_response.reason = 'Forbidden'
+        http_response.request = Request()
+
+        session = botocore.session.get_session()
+        s3 = session.get_service('s3')
+        operation = s3.get_operation('ListObjects') # non-streaming operation
+
+        self.assertEqual(
+            response.get_response(session, operation, http_response)[1], 
+            { 'ResponseMetadata': {},
+              'Errors': [{'HostId': 'AAAAAAAAAAAAAAAAAAA',
+                          'Message': 'Access Denied',
+                          'Code': 'AccessDenied',
+                          'RequestId': 'XXXXXXXXXXXXXXXX'}],
+              'ResponseMetadata': {},
+              u'CommonPrefixes': [],
+              u'Contents': [],
+              }
+            )
+    def test_get_response_nonstreaming_ng(self):
+        http_response = Response()
+        http_response.headers = {
+            'content-type': 'application/xml',
+            'date': 'Sat, 08 Mar 2014 12:05:44 GMT',
+            'server': 'AmazonS3',
+            'transfer-encoding': 'chunked',
+            'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA',
+            'x-amz-request-id': 'XXXXXXXXXXXXXXXX'}
+        http_response.raw = six.BytesIO(XMLBODY2)
+        http_response.status_code = 200
+        http_response.reason = 'ok'
+        http_response.request = Request()
+
+        session = botocore.session.get_session()
+        s3 = session.get_service('s3')
+        operation = s3.get_operation('ListObjects') # non-streaming operation
+
+        self.assertEqual(
+            response.get_response(session, operation, http_response)[1], 
+            {u'CommonPrefixes': [],
+             u'Contents': [{u'ETag': '"00000000000000000000000000000000"',
+                            u'Key': 'test.png',
+                            u'LastModified': '2014-03-01T17:06:40.000Z',
+                            u'Owner': {u'DisplayName': 'dummy',
+                                       u'ID': 'AAAAAAAAAAAAAAAAAAA'},
+                            u'Size': 6702,
+                            u'StorageClass': 'STANDARD'}],
+             u'IsTruncated': False,
+             u'Marker': None,
+             u'MaxKeys': 1000,
+             u'Name': 'mybucket',
+             u'Prefix': None,
+             'ResponseMetadata': {}}
+            )
+