Repository: boto/botocore
Issue: Errors not being parsed for streaming responses

If an error occurs, we don't parse the body of a streaming response.  For example if an `s3.GetObject` fails because of an auth issue, we don't get the parsed error response:

```
>>> import botocore.session
>>> session = botocore.session.get_session()
>>> s3 = session.get_service('s3')
>>> endpoint = s3.get_endpoint('us-east-1')
>>> operation = s3.get_operation('GetObject')
>>> response = operation.call(endpoint, bucket='testbucket', key='testkey')
>>> response[0].status_code
403
>>> response[1]
{u'Body': <botocore.vendored.requests.packages.urllib3.response.HTTPResponse object at 0x10c3747d0>, u'ContentType': 'application/xml'}
```

I can read the body to get the raw XML error:

```
>>> response[1]['Body'].read()
'<?xml version="1.0" encoding="UTF-8"?>\\n<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message><StringToSignBytes>sequence of bytes</StringToSignBytes><RequestId>requestid</RequestId><HostId>hostid</HostId><SignatureProvided>signature</SignatureProvided><StringToSign>string to sign</StringToSign><AWSAccessKeyId>access key</AWSAccessKeyId></Error>'
```

This also makes it difficult for code that uses botocore to present error messages in this scenario.  For example, in the CLI, the error you get for an auth error with `get-object` is:

```
 aws s3api get-object --bucket bucket  --key key /tmp/asdfasdfasdfasdfsdf
A client error (Unknown) occurred when calling the GetObject operation: Unknown
```


Changed files:
tests/unit/test_response.py, botocore/response.py

diff --git a/botocore/response.py b/botocore/response.py
index b489b2f9f..240d5a323 100644
--- a/botocore/response.py
+++ b/botocore/response.py
@@ -464,12 +464,29 @@ def get_response(session, operation, http_response):
         content_type = content_type.split(';')[0]
         logger.debug('Content type from response: %s', content_type)
     if operation.is_streaming():
+        logger.debug(
+            "Response Headers:\\n%s",
+            '\\n'.join("%s: %s" % (k, v) for k, v in http_response.headers.items()))
         streaming_response = StreamingResponse(session, operation)
         streaming_response.parse(
             http_response.headers,
             StreamingBody(http_response.raw,
                           http_response.headers.get('content-length')))
-        return (http_response, streaming_response.get_value())
+        if http_response.ok:
+            return (http_response, streaming_response.get_value())
+        else:
+            xml_response = XmlResponse(session, operation)
+            body = streaming_response.get_value()['Body'].read()
+            # For streaming response, response body might be binary data,
+            # so we log response body only when error happens.
+            logger.debug("Response Body:\\n%s", body)
+            if body:
+                try:
+                    xml_response.parse(body, encoding)
+                except xml.etree.cElementTree.ParseError as err:
+                    raise xml.etree.cElementTree.ParseError(
+                        "Error parsing XML response: %s\\nXML received:\\n%s" % (err, body))
+            return (http_response, xml_response.get_value())
     body = http_response.content
     if not http_response.request.method == 'HEAD':
         _validate_content_length(
diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py
index 436c7b9a0..1d70d2b7a 100644
--- a/tests/unit/test_response.py
+++ b/tests/unit/test_response.py
@@ -13,10 +13,28 @@
 
 from tests import unittest
 
-import mock
 import six
+import botocore
 from botocore import response
 from botocore.exceptions import IncompleteReadError
+from botocore.vendored.requests.models import Response, Request
+
+XMLBODY1 = (b'<?xml version="1.0" encoding="UTF-8"?><Error>'
+            b'<Code>AccessDenied</Code>'
+            b'<Message>Access Denied</Message>'
+            b'<RequestId>XXXXXXXXXXXXXXXX</RequestId>'
+            b'<HostId>AAAAAAAAAAAAAAAAAAA</HostId>'
+            b'</Error>')
+
+XMLBODY2 = (b'<?xml version="1.0" encoding="UTF-8"?>'
+            b'<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
+            b'<Name>mybucket</Name><Prefix></Prefix><Marker></Marker>'
+            b'<MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>'
+            b'<Contents><Key>test.png</Key><LastModified>2014-03-01T17:06:40.000Z</LastModified>'
+            b'<ETag>&quot;00000000000000000000000000000000&quot;</ETag><Size>6702</Size>'
+            b'<Owner><ID>AAAAAAAAAAAAAAAAAAA</ID>'
+            b'<DisplayName>dummy</DisplayName></Owner>'
+            b'<StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>')
 
 
 class TestStreamWrapper(unittest.TestCase):
@@ -39,3 +57,122 @@ class TestStreamWrapper(unittest.TestCase):
         stream = response.StreamingBody(body, content_length=10)
         with self.assertRaises(IncompleteReadError):
             stream.read()
+
+class TestGetResponse(unittest.TestCase):
+    def test_get_response_streaming_ok(self):
+        http_response = Response()
+        http_response.headers = {
+            'content-type': 'image/png',
+            'server': 'AmazonS3',
+            'AcceptRanges': 'bytes',
+            'transfer-encoding': 'chunked',
+            'ETag': '"00000000000000000000000000000000"',
+        }
+        http_response.raw = six.BytesIO(b'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00')
+
+        http_response.status_code = 200
+        http_response.reason = 'OK'
+
+        session = botocore.session.get_session()
+        s3 = session.get_service('s3')
+        operation = s3.get_operation('GetObject')
+
+        res = response.get_response(session, operation, http_response)
+        self.assertTrue(isinstance(res[1]['Body'], response.StreamingBody))
+        self.assertEqual(res[1]['ETag'],
+                         '"00000000000000000000000000000000"')
+
+    def test_get_response_streaming_ng(self):
+        http_response = Response()
+        http_response.headers = {
+            'content-type': 'application/xml',
+            'date': 'Sat, 08 Mar 2014 12:05:44 GMT',
+            'server': 'AmazonS3',
+            'transfer-encoding': 'chunked',
+            'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA',
+            'x-amz-request-id': 'XXXXXXXXXXXXXXXX'}
+        http_response.raw = six.BytesIO(XMLBODY1)
+        http_response.status_code = 403
+        http_response.reason = 'Forbidden'
+
+        session = botocore.session.get_session()
+        s3 = session.get_service('s3')
+        operation = s3.get_operation('GetObject') # streaming operation
+
+        self.assertEqual(
+            response.get_response(session, operation, http_response)[1], 
+            {u'Body': None,
+              'Errors': [{'HostId': 'AAAAAAAAAAAAAAAAAAA',
+                          'Message': 'Access Denied',
+                          'Code': 'AccessDenied',
+                          'RequestId': 'XXXXXXXXXXXXXXXX'}],
+              'ResponseMetadata': {},
+             u'Metadata': {}}
+            )
+
+    def test_get_response_nonstreaming_ok(self):
+        http_response = Response()
+        http_response.headers = {
+            'content-type': 'application/xml',
+            'date': 'Sun, 09 Mar 2014 02:55:43 GMT',
+            'server': 'AmazonS3',
+            'transfer-encoding': 'chunked',
+            'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA',
+            'x-amz-request-id': 'XXXXXXXXXXXXXXXX'}
+        http_response.raw = six.BytesIO(XMLBODY1)
+        http_response.status_code = 403
+        http_response.reason = 'Forbidden'
+        http_response.request = Request()
+
+        session = botocore.session.get_session()
+        s3 = session.get_service('s3')
+        operation = s3.get_operation('ListObjects') # non-streaming operation
+
+        self.assertEqual(
+            response.get_response(session, operation, http_response)[1], 
+            { 'ResponseMetadata': {},
+              'Errors': [{'HostId': 'AAAAAAAAAAAAAAAAAAA',
+                          'Message': 'Access Denied',
+                          'Code': 'AccessDenied',
+                          'RequestId': 'XXXXXXXXXXXXXXXX'}],
+              'ResponseMetadata': {},
+              u'CommonPrefixes': [],
+              u'Contents': [],
+              }
+            )
+    def test_get_response_nonstreaming_ng(self):
+        http_response = Response()
+        http_response.headers = {
+            'content-type': 'application/xml',
+            'date': 'Sat, 08 Mar 2014 12:05:44 GMT',
+            'server': 'AmazonS3',
+            'transfer-encoding': 'chunked',
+            'x-amz-id-2': 'AAAAAAAAAAAAAAAAAAA',
+            'x-amz-request-id': 'XXXXXXXXXXXXXXXX'}
+        http_response.raw = six.BytesIO(XMLBODY2)
+        http_response.status_code = 200
+        http_response.reason = 'ok'
+        http_response.request = Request()
+
+        session = botocore.session.get_session()
+        s3 = session.get_service('s3')
+        operation = s3.get_operation('ListObjects') # non-streaming operation
+
+        self.assertEqual(
+            response.get_response(session, operation, http_response)[1], 
+            {u'CommonPrefixes': [],
+             u'Contents': [{u'ETag': '"00000000000000000000000000000000"',
+                            u'Key': 'test.png',
+                            u'LastModified': '2014-03-01T17:06:40.000Z',
+                            u'Owner': {u'DisplayName': 'dummy',
+                                       u'ID': 'AAAAAAAAAAAAAAAAAAA'},
+                            u'Size': 6702,
+                            u'StorageClass': 'STANDARD'}],
+             u'IsTruncated': False,
+             u'Marker': None,
+             u'MaxKeys': 1000,
+             u'Name': 'mybucket',
+             u'Prefix': None,
+             'ResponseMetadata': {}}
+            )
+