Repository: DataBiosphere/toil

Issue:
409 conflict in _bindBucket
http://jenkins.cgcloud.info/job/toil-pull-requests/1699/testReport/junit/src.toil.test.jobStores.jobStoreTest/AWSJobStoreTest/test_testImportExportFile__otherCls_awsjobstoretest__size_one/

```
Error Message

S3ResponseError: S3ResponseError: 409 Conflict <?xml version="1.0" encoding="UTF-8"?> <Error><Code>OperationAborted</Code><Message>A conflicting conditional operation is currently in progress against this resource. Please try again.</Message><RequestId>4147B3737B28C799</RequestId><HostId>mRwjJPIqSZgMNgaqqDnkK7xl9t3HZBGs1Ttm25ODYkjRfnMIyPIS0ZsWDtHBQa/dIV5h0aaNG2o=</HostId></Error>
Stacktrace

src/toil/test/jobStores/jobStoreTest.py:104: in setUp
    self.master.initialize(self.config)
src/toil/jobStores/aws/jobStore.py:204: in initialize
    self._bind(create=True)
src/toil/jobStores/aws/jobStore.py:232: in _bind
    versioning=True)
src/toil/jobStores/aws/jobStore.py:677: in _bindBucket
    bucket.configure_versioning(versioning)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Bucket: jobstore-test-51045adb-8e94-4ae5-9ce6-795d95ed12e3--files>
versioning = True, mfa_delete = False, mfa_token = None, headers = None

    def configure_versioning(self, versioning, mfa_delete=False,
                             mfa_token=None, headers=None):
        """
            Configure versioning for this bucket.

            ..note:: This feature is currently in beta.

            :type versioning: bool
            :param versioning: A boolean indicating whether version is
                enabled (True) or disabled (False).

            :type mfa_delete: bool
            :param mfa_delete: A boolean indicating whether the
                Multi-Factor Authentication Delete feature is enabled
                (True) or disabled (False).  If mfa_delete is enabled then
                all Delete operations will require the token from your MFA
                device to be passed in the request.

            :type mfa_token: tuple or list of strings
            :param mfa_token: A tuple or list consisting of the serial
                number from the MFA device and the current value of the
                six-digit token associated with the device.  This value is
                required when you are changing the status of the MfaDelete
                property of the bucket.
            """
        if versioning:
            ver = 'Enabled'
        else:
            ver = 'Suspended'
        if mfa_delete:
            mfa = 'Enabled'
        else:
            mfa = 'Disabled'
        body = self.VersioningBody % (ver, mfa)
        if mfa_token:
            if not headers:
                headers = {}
            provider = self.connection.provider
            headers[provider.mfa_header] = ' '.join(mfa_token)
        response = self.connection.make_request('PUT', self.name, data=body,
                query_args='versioning', headers=headers)
        body = response.read()
        if response.status == 200:
            return True
        else:
            raise self.connection.provider.storage_response_error(
>               response.status, response.reason, body)
E           S3ResponseError: S3ResponseError: 409 Conflict
E           <?xml version="1.0" encoding="UTF-8"?>
E           <Error><Code>OperationAborted</Code><Message>A conflicting conditional operation is currently in progress against this resource. Please try again.</Message><RequestId>4147B3737B28C799</RequestId><HostId>mRwjJPIqSZgMNgaqqDnkK7xl9t3HZBGs1Ttm25ODYkjRfnMIyPIS0ZsWDtHBQa/dIV5h0aaNG2o=</HostId></Error>
```



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py
--- a/src/toil/jobStores/aws/jobStore.py
+++ b/src/toil/jobStores/aws/jobStore.py
@@ -664,7 +664,8 @@ def _bindBucket(self, bucket_name, create=False, block=True, versioning=False):
         def bucket_creation_pending(e):
             # https://github.com/BD2KGenomics/toil/issues/955
             # https://github.com/BD2KGenomics/toil/issues/995
-            return (isinstance(e, S3CreateError)
+            # https://github.com/BD2KGenomics/toil/issues/1093
+            return (isinstance(e, (S3CreateError, S3ResponseError))
                     and e.error_code in ('BucketAlreadyOwnedByYou', 'OperationAborted'))
 
         for attempt in retry_s3(predicate=bucket_creation_pending):


Test patch: