Repository: boto/boto

Issue:
Retry 502 errors
My project has recently seen a rash of transient 502 Bad Gateway and 504 Gateway Timeout errors from the SQS api. Currently the exponential back off mechanism in AWSAuthConnection._mexe is only retrying on 500s and 503s. I believe that 502s and 504s would be appropriate to retry as well. Here's a trivial patch:

```
diff --git a/boto/connection.py b/boto/connection.py
index ed57fc4..18bca5d 100644
--- a/boto/connection.py
+++ b/boto/connection.py
@@ -901,7 +901,7 @@ class AWSAuthConnection(object):
                             boto.log.debug(msg)
                         time.sleep(next_sleep)
                         continue
-                if response.status == 500 or response.status == 503:
+                if response.status in (500, 502, 503, 504):
                     msg = 'Received %d response.  ' % response.status
                     msg += 'Retrying in %3.1f seconds' % next_sleep
                     boto.log.debug(msg)
```

Retry 502 errors
My project has recently seen a rash of transient 502 Bad Gateway and 504 Gateway Timeout errors from the SQS api. Currently the exponential back off mechanism in AWSAuthConnection._mexe is only retrying on 500s and 503s. I believe that 502s and 504s would be appropriate to retry as well. Here's a trivial patch:

```
diff --git a/boto/connection.py b/boto/connection.py
index ed57fc4..18bca5d 100644
--- a/boto/connection.py
+++ b/boto/connection.py
@@ -901,7 +901,7 @@ class AWSAuthConnection(object):
                             boto.log.debug(msg)
                         time.sleep(next_sleep)
                         continue
-                if response.status == 500 or response.status == 503:
+                if response.status in (500, 502, 503, 504):
                     msg = 'Received %d response.  ' % response.status
                     msg += 'Retrying in %3.1f seconds' % next_sleep
                     boto.log.debug(msg)
```



Discussion:

Resolving patch:
diff --git a/boto/connection.py b/boto/connection.py
--- a/boto/connection.py
+++ b/boto/connection.py
@@ -901,7 +901,7 @@ def _mexe(self, request, sender=None, override_num_retries=None,
                             boto.log.debug(msg)
                         time.sleep(next_sleep)
                         continue
-                if response.status == 500 or response.status == 503:
+                if response.status in [500, 502, 503, 504]:
                     msg = 'Received %d response.  ' % response.status
                     msg += 'Retrying in %3.1f seconds' % next_sleep
                     boto.log.debug(msg)


Test patch:
diff --git a/tests/unit/s3/test_key.py b/tests/unit/s3/test_key.py
--- a/tests/unit/s3/test_key.py
+++ b/tests/unit/s3/test_key.py
@@ -114,6 +114,38 @@ def test_400_timeout(self, sleep_mock):
 
         self.assertTrue(k.should_retry.count, 1)
 
+    @mock.patch('time.sleep')
+    def test_502_bad_gateway(self, sleep_mock):
+        weird_timeout_body = "<Error><Code>BadGateway</Code></Error>"
+        self.set_http_response(status_code=502, body=weird_timeout_body)
+        b = Bucket(self.service_connection, 'mybucket')
+        k = b.new_key('test_failure')
+        fail_file = StringIO('This will pretend to be chunk-able.')
+
+        k.should_retry = counter(k.should_retry)
+        self.assertEqual(k.should_retry.count, 0)
+
+        with self.assertRaises(BotoServerError):
+            k.send_file(fail_file)
+
+        self.assertTrue(k.should_retry.count, 1)
+
+    @mock.patch('time.sleep')
+    def test_504_gateway_timeout(self, sleep_mock):
+        weird_timeout_body = "<Error><Code>GatewayTimeout</Code></Error>"
+        self.set_http_response(status_code=504, body=weird_timeout_body)
+        b = Bucket(self.service_connection, 'mybucket')
+        k = b.new_key('test_failure')
+        fail_file = StringIO('This will pretend to be chunk-able.')
+
+        k.should_retry = counter(k.should_retry)
+        self.assertEqual(k.should_retry.count, 0)
+
+        with self.assertRaises(BotoServerError):
+            k.send_file(fail_file)
+
+        self.assertTrue(k.should_retry.count, 1)
+
 
 if __name__ == '__main__':
     unittest.main()